From 0034d91b88bb5cf61d2f41affc64f481f2114c3a Mon Sep 17 00:00:00 2001 From: Ricardo Todling Date: Thu, 8 Oct 2026 12:39:58 -0400 Subject: [PATCH 1/9] Adding JEDI-atmens capability - driven by SWELL --- .../GEOSdas_App/jedi/etc/JEDIatmensConfig.csh | 19 ++ .../GEOSdas_App/jedi/etc/jedi_acquire_ioda.j | 2 +- .../GEOSdas_App/jedi/etc/jedi_eda.yaml | 263 ++++++++++++++++ .../GEOSdas_App/jedi/etc/jedi_getkf.yaml | 292 ++++++++++++++++++ .../GEOSdas_App/jedi/jedi_driver.csh | 9 +- .../GEOSdas_App/jedi/jedi_driver_atmens.csh | 170 ++++++++++ .../GEOSdas_App/jedi/jedi_gsi2ioda.csh | 2 +- .../GEOSdas_App/jedi/jedi_run.csh | 89 +++--- .../GEOSdas_App/jedi/jedi_run_atmens.csh | 186 +++++++++++ .../GEOSdas_App/jedi/jedi_set.csh | 25 +- .../GEOSdas_App/jedi/setup_aanajedi.pl | 1 + .../NCEP_Etc/NCEP_enkf/scripts/gmao/atm_ens.j | 45 ++- 12 files changed, 1040 insertions(+), 63 deletions(-) create mode 100644 src/Applications/GEOSdas_App/jedi/etc/JEDIatmensConfig.csh create mode 100644 src/Applications/GEOSdas_App/jedi/etc/jedi_eda.yaml create mode 100644 src/Applications/GEOSdas_App/jedi/etc/jedi_getkf.yaml create mode 100755 src/Applications/GEOSdas_App/jedi/jedi_driver_atmens.csh create mode 100755 src/Applications/GEOSdas_App/jedi/jedi_run_atmens.csh diff --git a/src/Applications/GEOSdas_App/jedi/etc/JEDIatmensConfig.csh b/src/Applications/GEOSdas_App/jedi/etc/JEDIatmensConfig.csh new file mode 100644 index 00000000..5355e526 --- /dev/null +++ b/src/Applications/GEOSdas_App/jedi/etc/JEDIatmensConfig.csh @@ -0,0 +1,19 @@ +# This is used to either defined JEDI-ATMENS-specific settings or +# override env definitions from JEDIanaConfig +# SLURM specials + +# Top options +setenv JEDI_RUN_EAANA 1 # run ensemble analysis +setenv JEDI_MKIAU 1 # calculates IAU output +setenv JEDI_MKIAU_CUBED 1 # 0=use lat-lon ana/bkg; 1=use cubed ana/bkg (no remap) + +# Details ... +setenv JEDI_OBS_OPT 1 # 1= point to xexp-like set (data in tar-balls; data from existing exp) + # 2= point to existing set of ncdiag-ioda-converted set (swell/.../DATE/geos_atmosphere) + # 3= generate on the fly based on GSI (nc4) diags (TBD) + +setenv SWELL_INSTALL /discover/nobackup/projects/gmao/advda/rtodling/JEDI1/2026/SWELL/Jul/ +setenv JEDI_ROOT /discover/nobackup/projects/gmao/advda/swell/JediBundles/fv3_soca_SLES15_08142026/build-intel-release/ + +setenv JEDI_RUN_ANA 0 + diff --git a/src/Applications/GEOSdas_App/jedi/etc/jedi_acquire_ioda.j b/src/Applications/GEOSdas_App/jedi/etc/jedi_acquire_ioda.j index 260708bc..9e772696 100644 --- a/src/Applications/GEOSdas_App/jedi/etc/jedi_acquire_ioda.j +++ b/src/Applications/GEOSdas_App/jedi/etc/jedi_acquire_ioda.j @@ -4,5 +4,5 @@ #SBATCH --time=1:00:00 cd $ACQWORK -acquire -v -rc $JEDIETC/jedi_ioda.acq -d $ACQWORK -s $FVHOME/spool -ssh $NYMD $NHMS 060000 1 +acquire -v -rc $JEDIETC/jedi_ioda.acq -d $ACQWORK -s $FVHOME/spool -ssh $NYMDB $NHMSB 060000 1 exit diff --git a/src/Applications/GEOSdas_App/jedi/etc/jedi_eda.yaml b/src/Applications/GEOSdas_App/jedi/etc/jedi_eda.yaml new file mode 100644 index 00000000..f6ae97db --- /dev/null +++ b/src/Applications/GEOSdas_App/jedi/etc/jedi_eda.yaml @@ -0,0 +1,263 @@ +# What is the experiment id? +experiment_id: ${EXPID}-eda_atmos + +# What is the experiment root (the directory where the experiment will be stored)? +experiment_root: $JEDIWORK + +# What is the time of the first cycle (middle of the window)? +start_cycle_point: '$JEDI_ISO_DATE_ANA' + +# What is the time of the final cycle (middle of the window)? +final_cycle_point: '$JEDI_ISO_DATE_ANA' + +# List of models in this experiment +model_components: +- geos_atmosphere + +# Set the Cylc runahead limit: the maximum number of cycles that may be active ahead of the current cycle (e.g. P1: up to 1 cycle ahead, P3: up to 3 cycles ahead, default P4). +runahead_limit: P2 + +# What experiment_id should r2d2 reference for experiment? +r2d2_experiment_id: ${EXPID}-eda_atmos-2ccaa7cf + +# Server/profile name in ~/.swell/r2d2_credentials.yaml (e.g. 'gmao_server'). Leave empty if credentials are at the root level. +r2d2_server: + +# Datastore name passed to R2D2 fetch and store operations (e.g. a Discover directory store or an S3 bucket store). Run scripts/discover_r2d2_datastores.py to list available datastores. Leave empty to let R2D2 pick the highest-priority writable datastore for your compute host. +r2d2_datastore: + +# Skip registering and storing results of this experiment in R2D2? +skip_r2d2: false + +# Record of the suite being executed +suite_to_run: eda_atmos + +# Dry-run option for comparing configs. +mock_experiment: false + +# Do you want to use an existing JEDI build or create a new build? +jedi_build_method: use_existing + +# What is the path to the existing JEDI build directory? +existing_jedi_build_directory: $JEDI_ROOT/ + +# What is the path to the existing JEDI source code directory? +existing_jedi_source_directory: $JEDI_ROOT/../ + +# Configurations for the model components. +models: + + # Configuration for the geos_atmosphere model component. + geos_atmosphere: + + # Enter the cycle times for this model. + cycle_times: + - T00 + - T06 + - T12 + - T18 + + # Do you want to use cycling VarBC option? + cycling_varbc: false + + # What is the duration for the data assimilation window? + window_length: PT6H + + # Do you want to use a 3D or 4D (including FGAT) window? + window_type: 3D + + # What is the name of the name of the experiment providing the backgrounds? + background_experiment: ${EXPID} + + # What is the horizontal resolution for the forecast model and backgrounds? + horizontal_resolution: '91' + + # How long before the middle of the analysis window did the background providing forecast begin? + background_time_offset: PT9H + + # What is the path to the GEOS X-backgrounds directory? + geos_x_background_directory: ${JEDIWORK}/archive/ + + # Perform check for observations? Set to false for debugging purposes. + check_for_obs: true + + # What is the path to the CRTM coefficient files? + crtm_coeff_dir: $JEDI_CRTM_COEFFS + + # What is the path to the Swell formatted observing system records? + observing_system_records_path: None + + # Which observations do you want to include? + observations: + - aircraft_temperature + - aircraft_wind + - amsr2_gcom-w1 + - amsua_aqua + - amsua_metop-b + - amsua_metop-c + - amsua_n15 + - amsua_n18 + - amsua_n19 + - atms_n20 + - atms_npp + - avhrr3_metop-b + - avhrr3_n18 + - avhrr3_n19 + - gmi_gpm + - gps + - mhs_metop-b + - mhs_metop-c + - mhs_n19 + - mls55_aura + - omi_aura + - ompsnm_npp + - pibal + - satwind + - scatwind + - sfcship + - sfc + - sondes + - ssmis_f17 +# cannnot properly perturb obs for chan-correlated data +# - airs_aqua +# - cris-fsr_n20 +# - cris-fsr_npp +# - iasi_metop-b +# - iasi_metop-c + + # What number of processors do you wish to use in the x-direction? + npx_proc: 4 + + # What number of processors do you wish to use in the y-direction? + npy_proc: 5 + + # What is the vertical resolution for the forecast model and background? + vertical_resolution: '72' + + # What is the rejection fraction for obs thinning? + obs_thinning_rej_fraction: 0.8 + + # Provide the log naming convention (e.g. 'variational', 'fgat'). + comparison_log_type: ensmeanvariance + + # What are the analysis variables? + analysis_variables: + - eastward_wind + - northward_wind + - air_temperature + - water_vapor_mixing_ratio_wrt_moist_air + - air_pressure_at_surface + - air_pressure_levels + - cloud_liquid_ice + - cloud_liquid_water + - rain_water + - snow_water + - mole_fraction_of_ozone_in_air + - geopotential_at_surface + - fraction_of_ocean + - fraction_of_lake + - fraction_of_ice + - skin_temperature_at_surface + + # What value of gradient norm reduction for convergence? + gradient_norm_reduction: 0.001 + + # Which GSIBEC climatological or hybrid? + gsibec_configuration: cli_gsibec_configuration + + # Which data assimilation minimizer do you wish to use? + minimizer: DRPCG + + # How many number of latutides in GSIBEC grid? + gsibec_nlats: '91' + + # How many number of longitudes in GSIBEC grid? + gsibec_nlons: '144' + + # What number of iterations do you wish to use for each outer loop? Provide a list of integers the same length as the number of outer loops. + number_of_iterations: + - 50 + + # How many members comprise the ensemble? + ensemble_num_members: 32 + + # What is the path to the Swell Static files directory? + swell_static_files: $JEDI_STATIC_FILES + + # Enter obs perturbation amplitude for EDA: + obs_pert_amplitude: 0.5 + + # What is the number of processors per host? + perhost: + + # How long before the middle of the analysis window did the ensemble background providing forecast begin? + ebkg_time_offset: PT3H + + # What is the path to the GEOS X-backgrounds directory? + geos_x_ensemble_directory: ${JEDIWORK}/archive + + # What is the database providing the observations? + obs_experiment: ${EXPID} + + # Map observation names to their R2D2 providers. + observation_providers: {} + + # Provide a list of patterns that you wish to remove from the cycle directory. + clean_patterns: [] + + # What is the path to the GSI formatted observing system records? + observing_system_records_mksi_path: $FVROOT/etc/ + + # What is the GSI formatted observing system records tag? + observing_system_records_mksi_path_tag: v5.44.1 + + # Configure the diffstates specifications: [state1, state2] + diffstates_spec: + state1: + fn_input: geos.prior.mean.ceta.%yyyy%mm%dd_%hh%MM%ssz.nc4 + state2: + fn_input: analysis/mean/eda_atmos.ana.ceta.%yyyy%mm%dd_%hh%MM%ssz.nc4 + state_diff: + fn_output: analysis/mean/eda_atmos.inc_iter1 + grid_type: + - cs + - latlon + state_type: ensemble + + # Provide a path that contains observation files not in r2d2. + ioda_locations_not_in_r2d2: $JEDIWORK/obs + + # Configure the ensemble mean and variance specifications: + ensmeanvariance_spec: + - state: bkg + fn_input: ebkg/mem%mem%/geos.mem%mem%.%yyyy%mm%dd_%hh%MM%ssz.nc4 + fn_output_mean: geos.prior.mean + fn_output_variance: geos.prior.variance + grid_type: + - cs + - latlon + - state: analysis + fn_input: analysis/mem%mem%/eda_atmos.ana.ceta.%yyyy%mm%dd_%hh%MM%ssz.nc4 + fn_output_mean: analysis/mean/eda_atmos.ana + fn_output_variance: analysis/variance/eda_atmos.ana + grid_type: + - cs + - latlon + +# Generate JEDI executable YAML and exit? +generate_yaml_and_exit: false + +# What is the path to the user provided Swell Static Files directory? +swell_static_files_user: None + +# Use cached observation files if they already exist? +cache_fetch: true + +# Fetch observations directly from a public S3 bucket if they are available? +fetch_obs_from_public_s3: false + +# Datetime this file was created (auto added) +datetime_created: 20261006_115816Z + +# Computing platform to run the experiment +platform: nccs_discover_sles15 diff --git a/src/Applications/GEOSdas_App/jedi/etc/jedi_getkf.yaml b/src/Applications/GEOSdas_App/jedi/etc/jedi_getkf.yaml new file mode 100644 index 00000000..1777ea1d --- /dev/null +++ b/src/Applications/GEOSdas_App/jedi/etc/jedi_getkf.yaml @@ -0,0 +1,292 @@ +# What is the experiment id? +experiment_id: ${EXPID}-localensembleda + +# What is the experiment root (the directory where the experiment will be stored)? +experiment_root: $JEDIWORK + +# What is the time of the first cycle (middle of the window)? +start_cycle_point: '$JEDI_ISO_DATE_ANA' + +# What is the time of the final cycle (middle of the window)? +final_cycle_point: '$JEDI_ISO_DATE_ANA' + +# List of models in this experiment +model_components: +- geos_atmosphere + +# Set the Cylc runahead limit: the maximum number of cycles that may be active ahead of the current cycle (e.g. P1: up to 1 cycle ahead, P3: up to 3 cycles ahead, default P4). +runahead_limit: P4 + +# What experiment_id should r2d2 reference for experiment? +r2d2_experiment_id: ${EXPID}-localensembleda-668389b7 + +# Server/profile name in ~/.swell/r2d2_credentials.yaml (e.g. 'gmao_server'). Leave empty if credentials are at the root level. +r2d2_server: + +# Datastore name passed to R2D2 fetch and store operations (e.g. a Discover directory store or an S3 bucket store). Run scripts/discover_r2d2_datastores.py to list available datastores. Leave empty to let R2D2 pick the highest-priority writable datastore for your compute host. +r2d2_datastore: + +# Skip registering and storing results of this experiment in R2D2? +skip_r2d2: false + +# Record of the suite being executed +suite_to_run: localensembleda + +# Dry-run option for comparing configs. +mock_experiment: false + +# Do you want to use an existing JEDI build or create a new build? +jedi_build_method: use_existing + +# What is the path to the existing JEDI source code directory? +existing_jedi_source_directory: $JEDI_ROOT/../ + +# What is the path to the existing JEDI build directory? +existing_jedi_build_directory: $JEDI_ROOT + +# Configurations for the model components. +models: + + # Configuration for the geos_atmosphere model component. + geos_atmosphere: + + # Enter the cycle times for this model. + cycle_times: + - T00 + - T06 + - T12 + - T18 + + # Do you want to run localensembleda observer and solver together? + skip_ensemble_hofx: true + + # Do you want to use cycling VarBC option? + cycling_varbc: false + + # Enter number of packets in which ensemble observers should be computed. + ensemble_hofx_packets: 2 + + # Enter hofx strategy. + ensemble_hofx_strategy: parallel + + # Provide a path that contains observation files not in r2d2. + ioda_locations_not_in_r2d2: ${JEDIWORK}/obs + + # What number of processors do you wish to use in the x-direction? + npx_proc: 4 + + # What number of processors do you wish to use in the y-direction? + npy_proc: 4 + + # What is the horizontal resolution for the forecast model and backgrounds? + horizontal_resolution: '91' + + # What is the vertical resolution for the forecast model and background? + vertical_resolution: '72' + + # What is the duration for the data assimilation window? + window_length: PT6H + + # Do you want to use a 3D or 4D (including FGAT) window? + window_type: 3D + + # What are the analysis variables? + analysis_variables: + - eastward_wind + - northward_wind + - air_temperature + - water_vapor_mixing_ratio_wrt_moist_air + - air_pressure_at_surface + - air_pressure_levels + - cloud_liquid_ice + - cloud_liquid_water + - rain_water + - snow_water + - mole_fraction_of_ozone_in_air + - geopotential_at_surface + - fraction_of_ocean + - fraction_of_lake + - fraction_of_ice + - skin_temperature_at_surface + + # How many members comprise the ensemble? + ensemble_num_members: 32 + + # Configure the ensemble mean and variance specifications: + ensmeanvariance_spec: + - {} + + # Which observations do you want to include? + observations: + - aircraft_temperature + - aircraft_wind + - amsr2_gcom-w1 + - amsua_aqua + - amsua_metop-b + - amsua_metop-c + - amsua_n15 + - amsua_n18 + - amsua_n19 + - atms_n20 + - atms_npp + - avhrr3_metop-b + - avhrr3_n18 + - avhrr3_n19 + - gps + - pibal + - satwind + - scatwind + - sfcship + - sfc + - sondes + + # What is the path to the Swell formatted observing system records? + observing_system_records_path: None + + # Provide the log naming convention (e.g. 'variational', 'fgat'). + comparison_log_type: localensembleda + + # What is the path to the Swell Static files directory? + swell_static_files: $JEDI_STATIC_FILES + + # Which GSIBEC climatological or hybrid? + gsibec_configuration: cli_gsibec_configuration + + # How many number of latutides in GSIBEC grid? + gsibec_nlats: 91 + + # How many number of longitudes in GSIBEC grid? + gsibec_nlons: 144 + + # How long before the middle of the analysis window did the background providing forecast begin? + background_time_offset: PT9H + + # What is the path to the CRTM coefficient files? + crtm_coeff_dir: $JEDI_CRTM_COEFFS + + # What is the database providing the observations? + obs_experiment: ${EXPID} + + # Map observation names to their R2D2 providers. + observation_providers: {} + + # What is the name of the name of the experiment providing the backgrounds? + background_experiment: ${EXPID} + + # Provide a list of patterns that you wish to remove from the cycle directory. + clean_patterns: + - '*.txt' + + # How long before the middle of the analysis window did the ensemble background providing forecast begin? + ebkg_time_offset: PT3H + + # What is the path to the GEOS X-backgrounds directory? + geos_x_ensemble_directory: ${ARCHIVE}/ + + # What is the path to the GSI formatted observing system records? + observing_system_records_mksi_path: None + + # Perform check for observations? Set to false for debugging purposes. + check_for_obs: true + + # What is the rejection fraction for obs thinning? + obs_thinning_rej_fraction: 0.75 + + # What is the GSI formatted observing system records tag? + observing_system_records_mksi_path_tag: v5.44.1 + + # Calculate ensemble mean only? + ensmean_only: false + + # Calculate ensemble mean and variance only? + ensmeanvariance_only: false + + # Specify the multiplicative prior inflation coefficient (0 inf]. + local_ensemble_inflation_mult: 1.1 + + # Specify the Relaxation To Prior Perturbation (RTPP) coefficient (0 1]. + local_ensemble_inflation_rtpp: 0.6 + + # Specify the Relaxation To Prior Spread (RTPS) coefficient (0 1]. + local_ensemble_inflation_rtps: 0.5 + + # Save the posterior ensemble members? + local_ensemble_save_posterior_ensemble: false + + # Save the posterior ensemble member increments? + local_ensemble_save_posterior_ensemble_increments: false + + # Save the posterior ensemble mean? + local_ensemble_save_posterior_mean: true + + # Save the posterior ensemble mean increment? + local_ensemble_save_posterior_mean_increment: true + + # Do posterior observer (OMA) for Local Ensemble DA? + local_ensemble_do_posterior_observer: false + + # Which local ensemble solver type should be implemented? + local_ensemble_solver: Deterministic GETKF + + # Use linear observer in local ensemble solver? + local_ensemble_use_linear_observer: true + + # Shall netCDF files from HofX output be skipped for OMB and OMA? + skip_hofx_output: false + + # What is the length scale for horizontal covariance localization? + horizontal_localization_lengthscale: 10000000.0 + + # What is the maximum number of observations to consider for horizontal covariance localization? + horizontal_localization_max_nobs: 1000 + + # Which localization scheme should be applied in the horizontal? + horizontal_localization_method: Horizontal Gaspari-Cohn + + # Should a log (base 10) transformation be applied to vertical coordinate when constructing vertical localization? + vertical_localization_apply_log_transform: true + + # Which localization scheme should be applied in the vertical? + vertical_localization_function: Gaspari Cohn + + # Which coordinate should be used in constructing vertical localization? + vertical_localization_ioda_vertical_coord: pressure + + # Which vertical coordinate group should be used in constructing vertical localization? + vertical_localization_ioda_vertical_coord_group: MetaData + + # What is the length scale for vertical covariance localization? + vertical_localization_lengthscale: 2 + + # What is the vertical localization unit for GETKF? + vertical_localization_unit: logp + + # What is the fraction of vertical retained variance for GETKF? + vertical_localization_frac_retained_variance: 0.95 + + # What localization scheme should be applied in constructing a vertical localization? + vertical_localization_method: Vertical localization + + # What is the number of processors per host? + perhost: 48 + + # Shall variational bc be changed to static bc in local ensemble DA yaml? + change_vbc_to_sbc: false + +# Generate JEDI executable YAML and exit? +generate_yaml_and_exit: false + +# What is the path to the user provided Swell Static Files directory? +swell_static_files_user: None + +# Use cached observation files if they already exist? +cache_fetch: true + +# Fetch observations directly from a public S3 bucket if they are available? +fetch_obs_from_public_s3: false + +# Datetime this file was created (auto added) +datetime_created: 20260928_132540Z + +# Computing platform to run the experiment +platform: nccs_discover_sles15 diff --git a/src/Applications/GEOSdas_App/jedi/jedi_driver.csh b/src/Applications/GEOSdas_App/jedi/jedi_driver.csh index 75d0e5f0..63100ca0 100755 --- a/src/Applications/GEOSdas_App/jedi/jedi_driver.csh +++ b/src/Applications/GEOSdas_App/jedi/jedi_driver.csh @@ -76,17 +76,16 @@ if ( !($?JEDI_IAU_OVERWRITE) ) setenv JEDI_IAU_OVERWRITE 0 if ( !($?JEDI_RUN_ADANA_TEST) ) setenv JEDI_RUN_ADANA_TEST 0 if ( !($?JEDI_RUN_ADANA) ) setenv JEDI_RUN_ADANA 0 if ( !($?JEDI_RUN_ANASA) ) setenv JEDI_RUN_ANASA 0 -if ( !($?JEDI_RUN_EAANA) ) setenv JEDI_RUN_EAANA 0 if ( !($?JEDI_OBS_OPT) ) setenv JEDI_OBS_OPT 0 if ( !($?JEDI_SWELLUSE) ) setenv JEDI_SWELLUSE 1 if ( !($?OFFLINE_IODA_DIR) ) setenv OFFLINE_IODA_DIR /dev/null if ( $JEDI_RUN_ADANA ) then - source $FVHOME/run/jedi/JEDIadanaConfig.csh + source $FVHOME/run/jedi/JEDIadanaConfig.csh else - source $FVHOME/run/jedi/JEDIanaConfig.csh - if ( $JEDI_RUN_EAANA ) then - source $FVHOME/run/jedi/JEDIatmensConfig.csh + source $FVHOME/run/jedi/JEDIanaConfig.csh + if ( -e $FVHOME/run/jedi/JEDIatmensConfig.csh ) then + source $FVHOME/run/jedi/JEDIatmensConfig.csh endif endif diff --git a/src/Applications/GEOSdas_App/jedi/jedi_driver_atmens.csh b/src/Applications/GEOSdas_App/jedi/jedi_driver_atmens.csh new file mode 100755 index 00000000..fa622b86 --- /dev/null +++ b/src/Applications/GEOSdas_App/jedi/jedi_driver_atmens.csh @@ -0,0 +1,170 @@ +#!/bin/csh + +setenv MYNAME jedi_driver.csh + +if ( ! $?FVHOME ) then + env + echo " ${MYNAME}: need FVHOME env" + exit (1) +endif + +if ( ! -d $FVHOME/run/jedi ) then + echo " ${MYNAME}: Nothing to do." +else + setenv JEDIDIR $FVHOME/run/jedi +endif + +if ( $#argv < 5 ) then + echo " " + echo " \\begin{verbatim} " + echo " " + echo " NAME " + echo " " + echo " $MYNAME - drive entire JEDI-based analysis for GEOS" + echo " " + echo " SYNOPSIS " + echo " " + echo " $MYNAME nymdb nhmsb nymda nhmsa fcoff " + echo " " + echo " nymdb - starting date of cycle" + echo " nhmsb - starting time of cycle" + echo " nymda - analysis date" + echo " nhmsa - analysis time" + echo " fcoff - hours offset from (nymdb,nhmsb)" + echo " " + echo " AUTHOR" + echo " Ricardo Todling (Ricardo.Todling@nasa.gov), NASA/GMAO " + echo " Initial version: 18Oct2020 by: R. Todling" + echo " Last modified: 28Jul2026 by: R. Todling" + echo " \\end{verbatim} " + echo " \\clearpage " + echo " ${MYNAME}: invalid arg list, aborting" + exit(1) +endif + +set nymdb = $1 +set nhmsb = $2 +set nymda = $3 +set nhmsa = $4 +set fcoff = $5 +set yyyya = `echo nymda | cut -c1-4` +set mma = `echo nymda | cut -c5-6` +set dda = `echo nymda | cut -c7-8` +set hhb = `echo $nhmsb | cut -c1-2` +set hha = `echo $nhmsa | cut -c1-2` +set yyyymmddhh = ${nymda}${hha} + +setenv NYMDA $nymda +setenv HHA $hha + +setenv FAILED 0 +if ( !($?EXPID) ) setenv FAILED 1 +if ( !($?FVWORK) ) setenv FAILED 1 + +if ( $FAILED ) then + env + echo " ${MYNAME}: not all required env vars defined" + exit (1) +endif + +# Defaults +if ( !($?JEDI_SET) ) setenv JEDI_SET 1 +if ( !($?JEDI_RUN) ) setenv JEDI_RUN 1 +if ( !($?JEDI_MKIAU)) setenv JEDI_MKIAU 0 +if ( !($?JEDI_IAU_OVERWRITE) ) setenv JEDI_IAU_OVERWRITE 1 +if ( !($?JEDI_RUN_ADANA_TEST) ) setenv JEDI_RUN_ADANA_TEST 0 +if ( !($?JEDI_RUN_ADANA) ) setenv JEDI_RUN_ADANA 0 +if ( !($?JEDI_RUN_ANASA) ) setenv JEDI_RUN_ANASA 0 +if ( !($?JEDI_RUN_EAANA) ) setenv JEDI_RUN_EAANA 0 +if ( !($?JEDI_OBS_OPT) ) setenv JEDI_OBS_OPT 0 +if ( !($?JEDI_SWELLUSE) ) setenv JEDI_SWELLUSE 1 +if ( !($?OFFLINE_IODA_DIR) ) setenv OFFLINE_IODA_DIR /dev/null + +source $FVHOME/run/jedi/JEDIanaConfig.csh +if ( -e $FVHOME/run/jedi/JEDIatmensConfig.csh ) then + source $FVHOME/run/jedi/JEDIatmensConfig.csh +else + echo "${MYNAME}: missing JEDIatmens Config" + exit(1) +endif + +if ( -e $FVWORK/.DONE_${MYNAME}.$yyyymmddhh ) then + echo "${MYNAME}: all done" + exit(0) +endif + +setenv JEDIWORK $FVWORK/jedi.$nymda.$nhmsa +if ( ! -d $JEDIWORK ) mkdir -p $JEDIWORK + +# Setup SWELL & IODA Files +# ======================== +if ( (! $JEDI_RUN_ANASA) && (! $JEDI_RUN_ADANA) ) then + if ( $JEDI_SWELLUSE ) then + jedi_swellset.csh $nymda $nhmsa $JEDIDIR $JEDIWORK + if ($status) then + echo "${MYNAME}: failed, aborting ..." + exit (1) + endif + + else + +# If here, IODA files must be available +# ------------------------------------- + if ( $OFFLINE_IODA_DIR == "/dev/null" || $OFFLINE_IODA_DIR == "/dev/null/" ) then + + # Convert GSI-nc4-diag files to IODA + # ---------------------------------- + if ( $JEDI_OBS_OPT == 3 ) then + jedi_gsi2ioda.csh $nymda $nhmsa $FVWORK $FVWORK $JEDIWORK + if ( $status ) then + echo "Trouble converting GSI output to IODA, aborting ..." + exit 1 + endif + endif + + else + set IODADIR = $OFFLINE_IODA_DIR/${nymda}T${hha}0000Z/geos_atmosphere + if ( ! -d $FVWORK/ioda.${nymda}_${hha}0000 ) mkdir $FVWORK/ioda.${nymda}_${hha}0000 + cd $FVWORK/ioda.${nymda}_${hha}0000 + ln -s $IODADIR/*.nc4 . + ln -s $IODADIR/*.txt . + cd - + endif + + endif +endif + +# Prepare env for analysis +# ------------------------ +if ( $JEDI_SET ) then + zeit_ci.x jedi_set + jedi_set.csh $nymdb $nhmsb $fcoff |& tee -a $FVWORK/$EXPID.jedi_set.log.${nymdb}_${hhb}z.txt + if ( $status ) then + echo " ${MYNAME}: jedi_set.csh signal failure, aborting ..." + exit (1) + endif + zeit_co.x jedi_set +endif + +# Run JEDI analysis +if ( $JEDI_RUN ) then + zeit_ci.x jedi_run + jedi_run_atmens.csh $nymda $nhmsa $FVWORK $FVWORK $JEDIWORK |& tee -a $FVWORK/$EXPID.jedi_run_atmens.log.${nymdb}_${hhb}z.txt + if ( $status ) then + echo " ${MYNAME}: jedi_run_atmens.csh signal failure, aborting ..." + exit (1) + endif + zeit_co.x jedi_run +endif + +if ( $fcoff ) then + touch $FVWORK/.DONE_${MYNAME}.$yyyymmddhh + echo " ${MYNAME}: Complete " + exit(0) +endif + +# If here, likely successful +# -------------------------- +touch $FVWORK/.DONE_${MYNAME}.$yyyymmddhh +echo " ${MYNAME}: Complete " +exit(0) diff --git a/src/Applications/GEOSdas_App/jedi/jedi_gsi2ioda.csh b/src/Applications/GEOSdas_App/jedi/jedi_gsi2ioda.csh index 801ae05d..2f2c1942 100755 --- a/src/Applications/GEOSdas_App/jedi/jedi_gsi2ioda.csh +++ b/src/Applications/GEOSdas_App/jedi/jedi_gsi2ioda.csh @@ -146,7 +146,7 @@ set phha = `echo $pnhmsa | cut -c1-2` endif # launch convert suite - swell launch $JEDIWORK/${EXPID}-convert_ncdiags/${EXPID}-convert_ncdiags-suite -b -t PT30S + swell launch $JEDIWORK/${EXPID}-convert_ncdiags/${EXPID}-convert_ncdiags-suite -b -t PT45S if ( $status ) then echo "Trouble converting GSI output to IODA, aborting ..." exit 1 diff --git a/src/Applications/GEOSdas_App/jedi/jedi_run.csh b/src/Applications/GEOSdas_App/jedi/jedi_run.csh index 2272fad5..c68f81f5 100755 --- a/src/Applications/GEOSdas_App/jedi/jedi_run.csh +++ b/src/Applications/GEOSdas_App/jedi/jedi_run.csh @@ -60,6 +60,7 @@ endif # Internal defaults # ----------------- +if ( !($?JEDI_ANAFREQ)) setenv JEDI_ANAFREQ 21600 if ( !($?JEDI_RUN_ADANA_TEST) ) setenv JEDI_RUN_ADANA_TEST 0 if ( !($?JEDI_RUN_BUMP) ) setenv JEDI_RUN_BUMP 0 if ( !($?JEDI_RUN_CNVENS) ) setenv JEDI_RUN_CNVENS 0 @@ -75,6 +76,15 @@ set hhb = `echo $nhmsb | cut -c1-2` set yyyymmddhh = ${nymdb}${hhb} setenv BYYYYYMMDDTHH0000Z ${nymdb}T${hhb}0000Z +set prvdate = `tick $nymdb $nhmsb -$JEDI_ANAFREQ` +set nymdp = $prvdate[1] +set nhmsp = $prvdate[2] +set yyyyp = `echo $nymdp | cut -c1-4` +set mmp = `echo $nymdp | cut -c5-6` +set ddp = `echo $nymdp | cut -c7-8` +set hhp = `echo $nhmsp | cut -c1-2` +setenv PYYYYYMMDDTHH0000Z ${nymdp}T${hhp}0000Z + set anadate = `tick $nymdb $nhmsb $JEDI_VAROFFSET` set nymda = $anadate[1] set nhmsa = $anadate[2] @@ -159,9 +169,6 @@ if ( ! -e $FVWORK/.DONE_jedi_run_ana.csh.$yyyymmddhh) then zeit_ci.x jedi_run set this = geosvar - if ( $JEDI_RUN_EAANA ) then - set this = geosens - endif if ( $fcoff ) then set this = geoshofx endif @@ -180,46 +187,35 @@ if ( ! -e $FVWORK/.DONE_jedi_run_ana.csh.$yyyymmddhh) then if (! -d inc ) mkdir inc endif - if ( $JEDI_RUN_EAANA ) then - if ( -e $FVHOME/run/jedi/jedi_run_eda.j ) then - vED -env $FVHOME/run/jedi/jedi_run_eda.j -o jedi_run_eda.j - sbatch -W jedi_run_eda.j - sleep 2 - else - echo " ${MYNAME}: cannot find jedi_run_eda.j, aborting ... " - exit (2) - endif + if ( -e $FVHOME/run/jedi/jedi_run_var.j ) then + vED -env $FVHOME/run/jedi/jedi_run_var.j -o jedi_run_var.j + sbatch -W jedi_run_var.j + sleep 2 else - if ( -e $FVHOME/run/jedi/jedi_run_var.j ) then - vED -env $FVHOME/run/jedi/jedi_run_var.j -o jedi_run_var.j - sbatch -W jedi_run_var.j - sleep 2 - else - if ( $fcoff ) then - $JEDI_FV3HOX_MPIRUN $JEDIBUILD/bin/fv3jedi_hofx_nomodel.x $MYCONF |& tee -a $FVWORK/$JEDIVARLOG -# $JEDI_FV3HOX_MPIRUN /discover/nobackup/projects/gmao/advda/rtodling/JEDI1/2026/24Jun/build-intel-release/bin/fv3jedi_hofx_nomodel.x $MYCONF |& tee -a $FVWORK/$JEDIVARLOG - if ( $status ) then - echo " ${MYNAME}: failed in VAR, aborting ..." - exit (1) - endif - else - $JEDI_FV3VAR_MPIRUN $JEDIBUILD/bin/fv3jedi_var.x $MYCONF |& tee -a $FVWORK/$JEDIVARLOG - if ( $status ) then - echo " ${MYNAME}: failed in VAR, aborting ..." - exit (1) - endif - endif - endif - set ichk = `grep NaN $FVWORK/$JEDIVARLOG | wc` - if ( $ichk[1] > 0 ) then - echo " ${MYNAME}: NaNs found in JEDI, failed in VAR, aborting ..." - exit (1) - endif - set lstinc = `ls *inc*nc4` - if ( ! $status ) /bin/mv *inc*nc4 ./inc # somehow datapath setting in yaml is not effective at inc part + if ( $fcoff ) then + $JEDI_FV3HOX_MPIRUN $JEDIBUILD/bin/fv3jedi_hofx_nomodel.x $MYCONF |& tee -a $FVWORK/$JEDIVARLOG +### $JEDI_FV3HOX_MPIRUN /discover/nobackup/projects/gmao/advda/rtodling/JEDI1/2026/24Jun/build-intel-release/bin/fv3jedi_hofx_nomodel.x $MYCONF |& tee -a $FVWORK/$JEDIVARLOG + if ( $status ) then + echo " ${MYNAME}: failed in VAR, aborting ..." + exit (1) + endif + else + $JEDI_FV3VAR_MPIRUN $JEDIBUILD/bin/fv3jedi_var.x $MYCONF |& tee -a $FVWORK/$JEDIVARLOG + if ( $status ) then + echo " ${MYNAME}: failed in VAR, aborting ..." + exit (1) + endif + endif + endif + set ichk = `grep NaN $FVWORK/$JEDIVARLOG | wc` + if ( $ichk[1] > 0 ) then + echo " ${MYNAME}: NaNs found in JEDI, failed in VAR, aborting ..." + exit (1) endif + set lstinc = `ls *inc*nc4` + if ( ! $status ) /bin/mv *inc*nc4 ./inc # somehow datapath setting in yaml is not effective at inc part zeit_co.x jedi_run - + if ( $fcoff ) then # archive hofx # ------------ @@ -358,7 +354,18 @@ foreach fn (`ls *tlapse.txt`) /bin/mv $fn $pfx.$BYYYYYMMDDTHH0000Z.tlapse.txt # time tag as current as # if JEDI output these end -tar cvf $FVWORK/$EXPID.jedi_vbc.${nymdb}_${hhb}z.tar *satbias*nc4 *aircraft*cov *.txt *.acftbias + +# Palliative for JEDI/UFO feature lacking +# JEDI does not write out acft-bias-cov, +# so copy from previous cycle to current +set acftcov = `ls aircraft_temperature.*acftbias_cov` +if ( ! -e aircraft_temperature.$BYYYYYMMDDTHH0000Z.acftbias_cov ) then + /bin/cp $JEDIWRK/obs/aircraft_temperature.$PYYYYYMMDDTHH0000Z.acftbias_cov \ + aircraft_temperature.$BYYYYYMMDDTHH0000Z.acftbias_cov +endif + +# Now tar up all bias and tlapse files +tar cvf $FVWORK/$EXPID.jedi_vbc.${nymdb}_${hhb}z.tar *satbias*nc4 *.txt *.acftbias* cd - # If here, likely successful diff --git a/src/Applications/GEOSdas_App/jedi/jedi_run_atmens.csh b/src/Applications/GEOSdas_App/jedi/jedi_run_atmens.csh new file mode 100755 index 00000000..8efb2e28 --- /dev/null +++ b/src/Applications/GEOSdas_App/jedi/jedi_run_atmens.csh @@ -0,0 +1,186 @@ +#!/bin/csh -x + +setenv MYNAME jedi_run_atmens.csh +setenv DRYRUN #echo + +if ( ! $?FVHOME ) then + env + echo " ${MYNAME}: need FVHOME env" + exit (1) +endif + +if ( ! -d $FVHOME/run/jedi ) then + echo " ${MYNAME}: Nothing to do." + exit (1) +else + setenv JEDIDIR $FVHOME/run/jedi +endif + +if ( $#argv < 5 ) then + echo " ${MYNAME}: invalid arg list, aborting" + exit(1) +endif + +set nymda = $1 +set nhmsa = $2 +set inpdir = $3 +set outdir = $4 +set jediwk = $5 +set hha = `echo $nhmsa | cut -c1-2` +set yyyymmddhh = ${nymda}${hha} + +setenv SYSFAILED 0 +if ( !($?LMOD_CMD) ) setenv SYSFAILED 1 +if ( !($?LMOD_SETTARG_CMD) ) setenv SYSFAILED 1 + +setenv FAILED 0 +if ( !($?EXPID) ) setenv FAILED 1 +if ( !($?FVHOME) ) setenv FAILED 1 +if ( !($?FVWORK) ) setenv FAILED 1 +if ( !($?SWELL_INSTALL) ) setenv FAILED 1 +if ( !($?TIMEINC) ) setenv FAILED 1 + +if ( $FAILED ) then + env + echo " ${MYNAME}: not all required env vars defined" + exit (1) +endif + +if (-e $FVWORK/.DONE_${MYNAME}.$yyyymmddhh ) then + echo " ${MYNAME}: already done" + exit(0) +endif + +if ( !($?JEDI_ROOT) ) setenv FAILED 1 +if ( !($?JEDI_CRTM_COEFFS) ) setenv FAILED 1 + +if ( $SYSFAILED ) then + env + echo " ${MYNAME}: not all SYSTEM required env vars defined" + exit (1) +endif +if ( $FAILED ) then + env + echo " ${MYNAME}: not all required env vars defined" + exit (1) +endif + +if ( ! $JEDI_RUN_EAANA ) then + echo "${MYNAME}: skipping jedi-atmens ..." + exit(0) +endif + +# set JEDI workdir +setenv JEDIETC $FVHOME/run/jedi/Config +setenv JEDIWORK $jediwk + +# define ISO analysis date/time +set yyyya = `echo $nymda | cut -c1-4` +set mma = `echo $nymda | cut -c5-6` +set dda = `echo $nymda | cut -c7-8` +set hha = `echo $nhmsa | cut -c1-2` +setenv JEDI_ISO_DATE_ANA "${yyyya}-${mma}-${dda}T${hha}:00:00Z" + +# set time of previous analysis (for bias stuff) +@ varlen = $TIMEINC * 60 +set panadate = `tick $nymda $nhmsa -$varlen` +set pnymda = $panadate[1] +set pnhmsa = $panadate[2] +set pyyyya = `echo $pnymda | cut -c1-4` +set pmma = `echo $pnymda | cut -c5-6` +set pdda = `echo $pnymda | cut -c7-8` +set phha = `echo $pnhmsa | cut -c1-2` + +# set cycle initial time +@ varoff = $VAROFFSET * 60 +set inidate = `tick $nymda $nhmsa -$varoff` +set nymdi = $inidate[1] +set nhmsi = $inidate[2] +set yyyyi = `echo $nymdi | cut -c1-4` +set mmi = `echo $nymdi | cut -c5-6` +set ddi = `echo $nymdi | cut -c7-8` +set hhi = `echo $nhmsi | cut -c1-2` + + # get positioned + cd $JEDIWORK + + # sanity check + if ( -e $JEDIETC/jedi_eda.yaml && \ + -e $JEDIETC/jedi_getkf.yaml ) then + echo "${MYNAME}: both EDA & GETKF are set to run ..." + echo "${MYNAME}: user should decide which, aborting ..." + exit(1) + endif + # prepare yaml to run JEDI-based atmos ens + set foundrc = 0 + if ( -e $JEDIETC/jedi_eda.yaml ) then + setenv SUITE eda_atmos + set myyaml = jedi_eda.yaml + set foundrc = 1 + echo "${MYNAME}: setting up to run JEDI-EDA" + endif + if ( -e $JEDIETC/jedi_getkf.yaml ) then + setenv SUITE localensembleda_2steps + set myyaml = jedi_getkf.yaml + set foundrc = 1 + echo "${MYNAME}: setting up to run JEDI-GETKF" + endif + if ( $foundrc ) then + vED -env $JEDIETC/$myyaml -o $myyaml + else + echo "${MYNAME}: no JEDI ensemble settings found ..." + exit(1) + endif + + ########################################################### + # The following is temporary: make sure the tarball of bkgs + # are available for SWELL (in a cycling exp the ensemble + # is readily available before tar'ing - but SWELL expects + # a tarball in the archive). + ########################################################### + if ( ! -d archive/$EXPID ) then + mkdir -p archive/$EXPID/atmens/Y$yyyyi/M$mmi + cd $FVHOME/atmens + mkdir $EXPID.atmens_ebkgx.${nymdi}_${hhi}z + mv ensbkgx $EXPID.atmens_ebkgx.${nymdi}_${hhi}z + tar cvf $JEDIWORK/archive/$EXPID/atmens/Y$yyyyi/M$mmi/$EXPID.atmens_ebkgx.${nymdi}_${hhi}z.tar \ + $EXPID.atmens_ebkgx.${nymdi}_${hhi}z + mv $EXPID.atmens_ebkgx.${nymdi}_${hhi}z/ensbkgx . + /bin/rm -r $EXPID.atmens_ebkgx.${nymdi}_${hhi}z + cd - + endif + + # create atmospheric ensemble suite + if ( -e $myyaml ) then + source $JEDIDIR/SWELLConfig.csh + if ( -d ~/cylc-run/${EXPID}-${SUITE}-suite ) then + /bin/rm -r ~/cylc-run/${EXPID}-${SUITE}-suite + endif + $DRYRUN swell create ${SUITE} --skip-r2d2 --override $myyaml + else + echo "Trouble finding $myyaml, aborting ..." + exit 1 + endif + + # launch convert suite + swell launch $JEDIWORK/${EXPID}-${SUITE}/${EXPID}-${SUITE}-suite -b -t PT30S + if ( $status ) then + echo "Trouble running JEDI atmos ensemble, aborting ..." + exit 1 + else +# if ( -d $JEDIWORK/${EXPID}-${SUITE}/run/${nymda}T${hha}0000Z/geos_atmosphere ) then +# if (! -d $outdir/ioda.${nymda}_${hha}0000 ) mkdir -p $outdir/ioda.${nymda}_${hha}0000 +# /bin/mv $JEDIWORK/${EXPID}-${SUITE}/run/${nymda}T${hha}0000Z/geos_atmosphere/*.nc4 \ +# $outdir/ioda.${nymda}_${hha}0000 +# /bin/mv $JEDIWORK/${EXPID}-${SUITE}/run/${nymda}T${hha}0000Z/geos_atmosphere/*.txt \ +# $outdir/ioda.${nymda}_${hha}0000 +# else +# echo "$JEDIWORK/${EXPID}-${SUITE}/run/${nymda}T${hha}0000Z/geos_atmosphere directory not found, aborting ..." +# exit 1 +# endif +# endif +#endif + +touch $FVWORK/.DONE_${MYNAME}.$yyyymmddhh +echo " ${MYNAME}: Complete " +exit(0) diff --git a/src/Applications/GEOSdas_App/jedi/jedi_set.csh b/src/Applications/GEOSdas_App/jedi/jedi_set.csh index 9e3a5148..0b71cf7b 100755 --- a/src/Applications/GEOSdas_App/jedi/jedi_set.csh +++ b/src/Applications/GEOSdas_App/jedi/jedi_set.csh @@ -247,21 +247,21 @@ endif # If so, feedback VarBC (from previous cycle) # ------------------------------------------- -if ( $JEDI_FEEDBACK_VARBC ) then +if ( $JEDI_FEEDBACK_VARBC || $JEDI_OBS_OPT == 1 ) then if ( -e $JEDIETC/VBC.BOOTSTRAP.DONE ) then # only when at least one JEDI cycle has completed - cd obs + cd $JEDIWRK # get tar-ball of varBC files from previous cycle setenv NYMDP $nymdp setenv NHMSP $nhmsp - setenv ACQWORK $JEDIWRK/obs + setenv ACQWORK $JEDIWRK vED -env $FVHOME/run/jedi/jedi_acquire_vbc.j -o jedi_acquire_vbc.j if ( $BATCH_SUBCMD == "sbatch" ) then sbatch -W -o jedi_vbc.log jedi_acquire_vbc.j else qsub -W block=true -o jedi_vbc.log jedi_acquire_vbc.j endif - set lstvbc = `ls *vbc*tar` + set lstvbc = `ls $JEDIWRK/*vbc*tar` if ($status) then echo "${MYNAME}: failed to retrieve vbc tar-ball" exit 1 @@ -269,7 +269,8 @@ if ( $JEDI_FEEDBACK_VARBC ) then #/bin/rm *satbias*.nc4 # unfold tar-ball and overwrite bias correction files with those from (own) previous cycle - tar xvf *vbc*tar + cd obs + tar xvf $JEDIWRK/*vbc*tar cd - endif @@ -289,6 +290,7 @@ cd obs end #endif cd - + # The following accommodates for the case when the satbias coeff and cov are in the same cd obs set acft = 0 @@ -330,6 +332,19 @@ if ( $acft ) then endif cd - +# To cope with SWELL get obs ... +# ------------------------------ +if ( ! -d $JEDIWRK/obs/${nymda}T${hha}0000Z/geos_atmosphere ) then + mkdir -p $JEDIWRK/obs/${nymda}T${hha}0000Z/geos_atmosphere +endif +cd $JEDIWRK/obs/${nymda}T${hha}0000Z/geos_atmosphere +ln -s ../../*.nc4 . +ln -s ../../*.txt . +ln -s ../../*.acftbias* . +#set acftbias_in = `ls aircraft_temperature.*.acftbias` +#ln -s $acftbias_in ${acftbias_in}_cov +cd - + # Build full yaml to run var, or grab existing yaml # ------------------------------------------------- set this = $JEDIETC/geosvar.${nymda}_${hha}z.yaml diff --git a/src/Applications/GEOSdas_App/jedi/setup_aanajedi.pl b/src/Applications/GEOSdas_App/jedi/setup_aanajedi.pl index c497ac92..0fd2931e 100755 --- a/src/Applications/GEOSdas_App/jedi/setup_aanajedi.pl +++ b/src/Applications/GEOSdas_App/jedi/setup_aanajedi.pl @@ -412,6 +412,7 @@ sub init { @rc2jediobs = qw ( 0observations.yaml abi_g16.yaml abi_g18.yaml + abi_g19.yaml aircraft_temperature.yaml aircraft_wind.yaml airs_aqua.yaml diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atm_ens.j b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atm_ens.j index 1b0c8c6d..b6f59843 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atm_ens.j +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atm_ens.j @@ -34,6 +34,22 @@ echo "atm_ens.j: cannot find $FVHOME/run/FVDAS_Run_Config, aborting ..." exit(1) endif + # Determines whether JEDI-based ensemble analysis will run + # in place of GSI-based ensemble (for now, experiments + # require a pure GSI-based initial cycle) + @ jedi_cycles = 0 + setenv JEDI_CAN_RUN_NOW 0 + if ( -d $FVHOME/run/jedi ) then + set jedi_cycles = `ls $FVHOME/run/jedi/Config/geosvar.*z.yaml | wc` + @ jedi_cycles = $jedi_cycles[1] # JEDI must have cycled twice for its ens to run (viz., vbc) + if ( $jedi_cycles > 1 && \ + -e $FVHOME/run/jedi/JEDIanaConfig.csh && \ + -e $FVHOME/run/jedi/JEDIatmensConfig.csh ) then + source $FVHOME/run/jedi/JEDIanaConfig.csh + source $FVHOME/run/jedi/JEDIatmensConfig.csh + setenv JEDI_CAN_RUN_NOW 1 + endif + endif # tell which side of the machine we are on ... uname -a @@ -55,6 +71,7 @@ if ( $?JOBGEN_PARTITION ) then setenv ATMENS_QNAME $JOBGEN_PARTITION endif + setenv BATCH_SUBCMD sbatch setenv GID >>>GID<<< setenv group_list "SBATCH -A $GID" setenv ARCH `uname -s` @@ -247,14 +264,16 @@ # Check for Variational Aircraft Bias Correction # ---------------------------------------------- - if ( (! $PREPQC ) && $ACFTBIAS == 2 ) then - set check_acfb = `echo $OBSCLASS | grep ncep_acftpfl_bufr` - if ($status) then - echo "Required ncep_acftpfl_bufr missing from OBSCLASS" - echo "Cannot perform Variational Aircraft Bias Correction" - echo "Aborting ..." - exit 1 - endif + if ( ! $JEDI_CAN_RUN_NOW ) then + if ( (! $PREPQC ) && $ACFTBIAS == 2 ) then + set check_acfb = `echo $OBSCLASS | grep ncep_acftpfl_bufr` + if ($status) then + echo "Required ncep_acftpfl_bufr missing from OBSCLASS" + echo "Cannot perform Variational Aircraft Bias Correction" + echo "Aborting ..." + exit 1 + endif + endif endif # Get date/time from restarts # --------------------------- @@ -428,7 +447,8 @@ # Run observer for mean and each ensemble member # ---------------------------------------------- - if ( $RUN_OBVSR || $DO_ATM_ENS ) then + if ( ! $JEDI_CAN_RUN_NOW ) then + if ( $RUN_OBVSR || $DO_ATM_ENS ) then zeit_ci.x obsvr obsvr_ensemble.csh $OBSCLASS $EXPID $anymd $anhms |& tee -a atm_ens.log if( $status ) then @@ -436,6 +456,7 @@ exit(1) endif zeit_co.x obsvr + endif endif # SAVING THE CONV DIAGS @@ -480,7 +501,11 @@ # ------------------------------------ if ( $RUN_EAANA || $DO_ATM_ENS ) then zeit_ci.x eana - atmos_eana.csh $EXPID $anymd $anhms |& tee -a atm_ens.log + if ( $JEDI_CAN_RUN_NOW ) then + jedi_driver_atmens.csh $nymdb $nhmsb $anymd $anhms 0 |& tee -a atm_ens.log + else + atmos_eana.csh $EXPID $anymd $anhms |& tee -a atm_ens.log + endif if( $status) then echo "eana failed" exit(1) From c4c3c309e29f68fe5a19ff1ca20857e4abbdfa02 Mon Sep 17 00:00:00 2001 From: Ricardo Todling Date: Thu, 8 Oct 2026 15:55:31 -0400 Subject: [PATCH 2/9] minor still working on atmens --- .../GEOSdas_App/jedi/etc/JEDIanaConfig.csh | 3 +++ src/Applications/GEOSdas_App/jedi/jedi_set.csh | 13 ++----------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/src/Applications/GEOSdas_App/jedi/etc/JEDIanaConfig.csh b/src/Applications/GEOSdas_App/jedi/etc/JEDIanaConfig.csh index 1b3ddfcd..cf84d4d9 100644 --- a/src/Applications/GEOSdas_App/jedi/etc/JEDIanaConfig.csh +++ b/src/Applications/GEOSdas_App/jedi/etc/JEDIanaConfig.csh @@ -52,3 +52,6 @@ setenv JEDI_INPUT @JEDI_INPUT # post setenv JEDI_CONCAT_IODA 0 + +# make sure ensemble analysis is no invoked +setenv JEDI_RUN_EAANA 0 diff --git a/src/Applications/GEOSdas_App/jedi/jedi_set.csh b/src/Applications/GEOSdas_App/jedi/jedi_set.csh index 0b71cf7b..d2878133 100755 --- a/src/Applications/GEOSdas_App/jedi/jedi_set.csh +++ b/src/Applications/GEOSdas_App/jedi/jedi_set.csh @@ -56,7 +56,6 @@ endif if ( !($?JEDI_ANAFREQ)) setenv JEDI_ANAFREQ 21600 if ( !($?JEDI_RUN_ADANA) ) setenv JEDI_RUN_ADANA 0 if ( !($?JEDI_RUN_ANASA) ) setenv JEDI_RUN_ANASA 0 -if ( !($?JEDI_RUN_EAANA) ) setenv JEDI_RUN_EAANA 0 if ( !($?JEDI_VAROFFSET)) setenv JEDI_VAROFFSET 10800 if ( !($?JEDI_VARWINDOW)) setenv JEDI_VARWINDOW 21600 if ( !($?MAPLFIX) ) setenv MAPLFIX 0 @@ -348,9 +347,6 @@ cd - # Build full yaml to run var, or grab existing yaml # ------------------------------------------------- set this = $JEDIETC/geosvar.${nymda}_${hha}z.yaml -if ( $JEDI_RUN_EAANA ) then - set this = $JEDIETC/geosens.${nymda}_${hha}z.yaml -endif if ( $fcoff ) then set this = $JEDIETC/geoshofx.${nymda}_${hha}z.yaml endif @@ -394,9 +390,6 @@ else # Construct full VAR yaml set this = geosvar - if ( $JEDI_RUN_EAANA ) then - set this = geosens - endif if ( $fcoff ) then set this = geoshofx endif @@ -412,7 +405,7 @@ endif # Acquire background ensemble (either for hybrid VAR or ensemble DA) setenv JEDI_GET_ENSBKG 0 if ( ! $fcoff ) then - if ( $JEDI_HYBRID || $JEDI_RUN_EAANA ) then + if ( $JEDI_HYBRID ) then if ( $JEDI_HYBRID == 1 ) then # lat-lon ensemble set ensdir = $FVHOME/atmens set bkgtyp = "bkg.eta" @@ -446,8 +439,7 @@ if ( ! $fcoff ) then endif # fcoff # Acquire background fields (unless running ensemble DA) -if ( (! $JEDI_RUN_EAANA) ) then - if ( ! -e $JEDIWRK/.DONE_JEDI_GET_BKG_${nymdb}_${nhmsb} ) then +if ( ! -e $JEDIWRK/.DONE_JEDI_GET_BKG_${nymdb}_${nhmsb} ) then cd bkg if ( $fcoff ) then setenv NYMD $nymdf # forecast initial date @@ -521,7 +513,6 @@ if ( (! $JEDI_RUN_EAANA) ) then endif endif # fcoff touch $JEDIWRK/.DONE_JEDI_GET_BKG_${nymdb}_${nhmsb} - endif endif # When applicable, retrieve ensemble background From 1f53ebb42e6a566b474b647ac0825cc3bf17a349 Mon Sep 17 00:00:00 2001 From: Ricardo Todling Date: Tue, 13 Oct 2026 13:43:27 -0400 Subject: [PATCH 3/9] Almost working ... --- src/Applications/GEOSdas_App/fvsetup | 2 + .../GEOSdas_App/jedi/jedi_driver.csh | 16 +- .../GEOSdas_App/jedi/jedi_driver_atmens.csh | 47 +++- .../GEOSdas_App/jedi/jedi_mkiau.csh | 222 +++++++++++------- .../GEOSdas_App/jedi/jedi_run_atmens.csh | 186 +++++++++++---- .../GEOSdas_App/jedi/jedi_swellset.csh | 85 ------- .../NCEP_Etc/NCEP_enkf/scripts/gmao/atm_ens.j | 18 +- .../NCEP_enkf/scripts/gmao/etc/AGCM.rc.tmpl | 2 +- .../scripts/gmao/etc/AtmEnsConfig.csh | 3 + .../NCEP_enkf/scripts/gmao/post_eana.csh | 53 +++-- 10 files changed, 378 insertions(+), 256 deletions(-) delete mode 100755 src/Applications/GEOSdas_App/jedi/jedi_swellset.csh diff --git a/src/Applications/GEOSdas_App/fvsetup b/src/Applications/GEOSdas_App/fvsetup index 0becd6bc..f793386f 100755 --- a/src/Applications/GEOSdas_App/fvsetup +++ b/src/Applications/GEOSdas_App/fvsetup @@ -7193,6 +7193,8 @@ sub arch_asens { \${PESTOROOT}%s/obs/Y%y4/M%m2/D%d2/H%h2/%s.imp%c_%c%c%c_ssu_%c%c%c.obs.%y4%m2%d2_%h2z+%y4%m2%d2_%h2z-%y4%m2%d2_%h2z.ods \${PESTOROOT}%s/obs/Y%y4/M%m2/D%d2/H%h2/%s.imp%c_%c%c%c_ssu_tirosn.obs.%y4%m2%d2_%h2z+%y4%m2%d2_%h2z-%y4%m2%d2_%h2z.ods \${PESTOROOT}%s/obs/Y%y4/M%m2/D%d2/H%h2/%s.imp%c_%c%c%c_tmi_trmm.obs.%y4%m2%d2_%h2z+%y4%m2%d2_%h2z-%y4%m2%d2_%h2z.ods +\${PESTOROOT}%s/obs/Y%y4/M%m2/D%d2/H%h2/%s.imp%c_%c%c%c_viirs-m_npp.obs.%y4%m2%d2_%h2z+%y4%m2%d2_%h2z-%y4%m2%d2_%h2z.ods +\${PESTOROOT}%s/obs/Y%y4/M%m2/D%d2/H%h2/%s.imp%c_%c%c%c_viirs-m_j%c.obs.%y4%m2%d2_%h2z+%y4%m2%d2_%h2z-%y4%m2%d2_%h2z.ods # #\${PESTOROOT}%s/obs/Y%y4/M%m2/D%d2/H%h2/%s.imp%c_%c%c%c_conv_%c%c%c_%c%c%c%c.obs.%y4%m2%d2_%h2z+%y4%m2%d2_%h2z-%y4%m2%d2_%h2z.ods #\${PESTOROOT}%s/obs/Y%y4/M%m2/D%d2/H%h2/%s.imp%c_%c%c%c_gps_bnd_%c%c%c_%c%c%c%c.obs.%y4%m2%d2_%h2z+%y4%m2%d2_%h2z-%y4%m2%d2_%h2z.ods diff --git a/src/Applications/GEOSdas_App/jedi/jedi_driver.csh b/src/Applications/GEOSdas_App/jedi/jedi_driver.csh index 63100ca0..6d2233de 100755 --- a/src/Applications/GEOSdas_App/jedi/jedi_driver.csh +++ b/src/Applications/GEOSdas_App/jedi/jedi_driver.csh @@ -84,9 +84,6 @@ if ( $JEDI_RUN_ADANA ) then source $FVHOME/run/jedi/JEDIadanaConfig.csh else source $FVHOME/run/jedi/JEDIanaConfig.csh - if ( -e $FVHOME/run/jedi/JEDIatmensConfig.csh ) then - source $FVHOME/run/jedi/JEDIatmensConfig.csh - endif endif if ( -e $FVWORK/.DONE_${MYNAME}.$yyyymmddhh ) then @@ -100,14 +97,6 @@ if ( ! -d $JEDIWORK ) mkdir -p $JEDIWORK # Setup SWELL & IODA Files # ======================== if ( (! $JEDI_RUN_ANASA) && (! $JEDI_RUN_ADANA) ) then - if ( $JEDI_SWELLUSE ) then - jedi_swellset.csh $nymda $nhmsa $JEDIDIR $JEDIWORK - if ($status) then - echo "${MYNAME}: failed, aborting ..." - exit (1) - endif - - else # If here, IODA files must be available # ------------------------------------- @@ -116,7 +105,7 @@ if ( (! $JEDI_RUN_ANASA) && (! $JEDI_RUN_ADANA) ) then # Convert GSI-nc4-diag files to IODA # ---------------------------------- if ( $JEDI_OBS_OPT == 3 ) then - jedi_gsi2ioda.csh $nymda $nhmsa $FVWORK $FVWORK $JEDIWORK + jedi_gsi2ioda.csh $nymda $nhmsa $FVWORK $FVWORK $JEDIWORK |& tee -a $FVWORK/$EXPID.jedi_g2i.log.${nymdb}_${hhb}z.txt if ( $status ) then echo "Trouble converting GSI output to IODA, aborting ..." exit 1 @@ -132,7 +121,6 @@ if ( (! $JEDI_RUN_ANASA) && (! $JEDI_RUN_ADANA) ) then cd - endif - endif endif # Prepare env for analysis @@ -158,7 +146,7 @@ if ( $JEDI_RUN ) then zeit_co.x jedi_run if ( $JEDI_MKIAU ) then zeit_ci.x jedi_mkiau - jedi_mkiau.csh $nymdb $nhmsb |& tee -a $FVWORK/$EXPID.jedi_miau.log.${nymdb}_${hhb}z.txt + jedi_mkiau.csh $nymdb $nhmsb "central" |& tee -a $FVWORK/$EXPID.jedi_miau.log.${nymdb}_${hhb}z.txt if ( $status ) then echo " ${MYNAME}: jedi_mkiau.csh signal failure, aborting ..." exit (1) diff --git a/src/Applications/GEOSdas_App/jedi/jedi_driver_atmens.csh b/src/Applications/GEOSdas_App/jedi/jedi_driver_atmens.csh index fa622b86..2e874994 100755 --- a/src/Applications/GEOSdas_App/jedi/jedi_driver_atmens.csh +++ b/src/Applications/GEOSdas_App/jedi/jedi_driver_atmens.csh @@ -1,6 +1,6 @@ -#!/bin/csh +#!/bin/csh -x -setenv MYNAME jedi_driver.csh +setenv MYNAME jedi_driver_atmens.csh if ( ! $?FVHOME ) then env @@ -91,6 +91,8 @@ endif if ( -e $FVWORK/.DONE_${MYNAME}.$yyyymmddhh ) then echo "${MYNAME}: all done" exit(0) +else + echo "${MYNAME}: Starting ... " endif setenv JEDIWORK $FVWORK/jedi.$nymda.$nhmsa @@ -99,14 +101,6 @@ if ( ! -d $JEDIWORK ) mkdir -p $JEDIWORK # Setup SWELL & IODA Files # ======================== if ( (! $JEDI_RUN_ANASA) && (! $JEDI_RUN_ADANA) ) then - if ( $JEDI_SWELLUSE ) then - jedi_swellset.csh $nymda $nhmsa $JEDIDIR $JEDIWORK - if ($status) then - echo "${MYNAME}: failed, aborting ..." - exit (1) - endif - - else # If here, IODA files must be available # ------------------------------------- @@ -131,7 +125,6 @@ if ( (! $JEDI_RUN_ANASA) && (! $JEDI_RUN_ADANA) ) then cd - endif - endif endif # Prepare env for analysis @@ -146,8 +139,23 @@ if ( $JEDI_SET ) then zeit_co.x jedi_set endif +# Set ensemble as in non-JEDI ATMENS +# TBD: the following lines belong to +# jedi_set +# ---------------------------------- +cd $FVWORK + if ( ! -d mem001 ) then + /bin/ln -s $FVHOME/atmens/mem* . +endif +set members = (`ls -d mem*`) + +# Handle mean and variance output +# ------------------------------- +if ( ! -d $FVWORK/updated_ens/ensmean ) mkdir -p $FVWORK/updated_ens/ensmean +if ( ! -d $FVWORK/updated_ens/ensrms ) mkdir -p $FVWORK/updated_ens/ensrms + # Run JEDI analysis -if ( $JEDI_RUN ) then +if ( $JEDI_RUN_EAANA ) then zeit_ci.x jedi_run jedi_run_atmens.csh $nymda $nhmsa $FVWORK $FVWORK $JEDIWORK |& tee -a $FVWORK/$EXPID.jedi_run_atmens.log.${nymdb}_${hhb}z.txt if ( $status ) then @@ -157,8 +165,23 @@ if ( $JEDI_RUN ) then zeit_co.x jedi_run endif +# The following needs parallelization +# ----------------------------------- +if ( $JEDI_MKIAU ) then + zeit_ci.x jedi_mkiau + foreach member ( $members ) + jedi_mkiau.csh $nymdb $nhmsb $member |& tee -a $FVWORK/$EXPID.jedi_miau.log.${nymdb}_${hhb}z.txt + if ( $status ) then + echo " ${MYNAME}: jedi_mkiau.csh signal failure for $member, aborting ..." + exit (1) + endif + end + zeit_co.x jedi_mkiau +endif + if ( $fcoff ) then touch $FVWORK/.DONE_${MYNAME}.$yyyymmddhh + echo " ${MYNAME}: Not yet ready for fcoff opt in atmens. " echo " ${MYNAME}: Complete " exit(0) endif diff --git a/src/Applications/GEOSdas_App/jedi/jedi_mkiau.csh b/src/Applications/GEOSdas_App/jedi/jedi_mkiau.csh index 08c3b8e1..3bd55294 100755 --- a/src/Applications/GEOSdas_App/jedi/jedi_mkiau.csh +++ b/src/Applications/GEOSdas_App/jedi/jedi_mkiau.csh @@ -8,9 +8,31 @@ endif setenv MYNAME jedi_mkiau.csh -if ( $#argv < 2 ) then - echo " ${MYNAME}: invalid arg list, aborting" - exit (1) +if ( $#argv < 3 ) then + echo " " + echo " \\begin{verbatim} " + echo " " + echo " NAME " + echo " " + echo " $MYNAME - run mkiau for JEDI analysis " + echo " " + echo " SYNOPSIS " + echo " " + echo " $MYNAME nymd nhms opt" + echo " " + echo " where " + echo " nymd - analysis synoptic date" + echo " nhms - analysis synoptic time" + echo " opt - opt = central, for central ADAS or" + echo " memNNN, for NNN-th member of JEDI ensemble" + echo " " + echo " AUTHOR" + echo " Ricardo Todling (Ricardo.Todling@nasa.gov), NASA/GMAO " + echo " Initial version: 18Oct2020 by: R. Todling" + echo " Last modified: 08Oct2026 by: R. Todling" + echo " \\end{verbatim} " + echo " \\clearpage " + exit(1) endif if ( !($?EXPID) ) setenv FAILED 1 @@ -31,6 +53,7 @@ endif set nymd0 = $1 set nhms0 = $2 +set opt = $3 set hh0 = `echo $nhms0 | cut -c1-2` set yyyymmddhh = $nymd0${hh0} @@ -38,9 +61,11 @@ set anadate = `tick $nymd0 $nhms0 $JEDI_VAROFFSET` set nymda = $anadate[1] set nhmsa = $anadate[2] -if ( -e $FVWORK/.DONE_${MYNAME}.$yyyymmddhh ) then +if ( -e $FVWORK/.DONE_${MYNAME}.$opt.$yyyymmddhh ) then echo "${MYNAME}: all done" exit(0) +else + echo "${MYNAME}: Starting ... " endif #source $FVROOT/bin/g5_modules @@ -49,18 +74,28 @@ endif # Defaults if ( !($?JEDI_VARWINDOW)) setenv JEDI_VARWINDOW 21600 if ( !($?JEDI_VARANAFRQ)) setenv JEDI_VARANAFRQ 3600 - setenv JEDIETC $FVHOME/run/jedi/Config -setenv JEDIWORK $FVWORK/jedi.$nymda.${nhmsa} - -# link bkg.eta files from FVHOME/recycle - GEOSdas workflow deletes some of these from fvwork -# when GSI is running -cd $JEDIWORK/bkg -touch .no_archiving -foreach fn ( `ls $FVHOME/recycle/*.bkg*eta_rst.*nc4` ) - set sfx = `basename $fn | cut -d. -f3-` - ln -sf $fn $EXPID.bkg.eta.$sfx -end + +if ( $opt == "central" ) then + setenv JEDIWORK $FVWORK/jedi.$nymda.${nhmsa} + # link bkg.eta files from FVHOME/recycle - GEOSdas workflow deletes some of these from fvwork + # when GSI is running + cd $JEDIWORK/bkg + touch .no_archiving + foreach fn ( `ls $FVHOME/recycle/*.bkg*eta_rst.*nc4` ) + set sfx = `basename $fn | cut -d. -f3-` + ln -sf $fn $EXPID.bkg.eta.$sfx + end +else + if ( -e $JEDIETC/jedi_eda.yaml ) then + setenv SUITE eda_atmos + endif + if ( -e $JEDIETC/jedi_getkf.yaml ) then + setenv SUITE localensembleda_2steps + endif + # mimic env configuration of central ADAS within each member directory + setenv JEDIWORK $FVWORK/jedi.$nymda.${nhmsa}/${EXPID}-${SUITE}/run/${nymda}T${nhmsa}Z/geos_atmosphere/analysis/$opt +endif # Create analysis from increment cd $JEDIWORK/inc @@ -111,92 +146,115 @@ set analst = `ls $EXPID.jedi_${anatyp}.*` cd $JEDIWORK -if ( ! -e IAU_EGRESS ) then - foreach anafn ( $analst ) - set ttag = `echo $anafn | cut -d. -f4` - set nymd = `echo $ttag | cut -c1-8` - set hhmm = `echo $ttag | cut -c10-13` - set nhms = ${hhmm}00 - set anafn = ana/$anafn - set bkgfn = bkg/$EXPID.${bkgtyp}.${nymd}_${hhmm}z.nc4 - set iaufn = iau/$EXPID.agcm_import_rst.${nymd}_${hhmm}z.nc4 - echo " Input Analysis file: $anafn" - echo " Input Background file: $bkgfn" - echo " Output IAU file: $iaufn" - # prepare RC - setenv ANADATE $nymd - setenv ANATIME $nhms - setenv ANAFNAME $anafn - setenv BKGFNAME $bkgfn - setenv AGCMIMPRST $iaufn - vED -env $JEDIETC/$thisrc -o mkiau.rc - - set nx = `echorc.x -rc mkiau.rc "NX"` - set ny = `echorc.x -rc mkiau.rc "NY"` - @ ncpus = $nx * $ny - - # generate IAU increment file - touch input.nml - $JEDI_MKIAU_MPIRUN -np $ncpus mkiau.x - - # clean up - #rm -f input.nml mkiau.rc - end -endif +# Only calculate IAU increment when running Central ADAS +# When running ensmeble IAU increments are only calculated +# after recentering is done on the analyses. +if ( $opt == "central" ) then + if ( ! -e IAU_EGRESS ) then + foreach anafn ( $analst ) + set ttag = `echo $anafn | cut -d. -f4` + set nymd = `echo $ttag | cut -c1-8` + set hhmm = `echo $ttag | cut -c10-13` + set nhms = ${hhmm}00 + set anafn = ana/$anafn + set bkgfn = bkg/$EXPID.${bkgtyp}.${nymd}_${hhmm}z.nc4 + set iaufn = iau/$EXPID.agcm_import_rst.${nymd}_${hhmm}z.nc4 + echo " Input Analysis file: $anafn" + echo " Input Background file: $bkgfn" + echo " Output IAU file: $iaufn" + # prepare RC + setenv ANADATE $nymd + setenv ANATIME $nhms + setenv ANAFNAME $anafn + setenv BKGFNAME $bkgfn + setenv AGCMIMPRST $iaufn + vED -env $JEDIETC/$thisrc -o mkiau.rc + + set nx = `echorc.x -rc mkiau.rc "NX"` + set ny = `echorc.x -rc mkiau.rc "NY"` + @ ncpus = $nx * $ny -# if so, fake 4d -if ( ! -d iau ) mkdir iau -cd iau/ -if ( $JEDI_HYBRID ) then - set lst = `ls *agcm_import_rst.*nc4` - if ( $#lst == 1 ) then - set nowdate = ( $nymd0 $nhms0 ) - set enddate = (`tick $nowdate[1] $nowdate[2] $JEDI_VARWINDOW`) - set notdone = 1 - while ( $notdone ) - set nymd = $nowdate[1]; set nhms = $nowdate[2]; set hhmm = `echo $nhms | cut -c1-4` - /bin/cp $lst[1] $EXPID.agcm_import_rst.${nymd}_${hhmm}z.nc4 - if ( "$nowdate" == "$enddate" ) then - set notdone = 0 - else - set nowdate = (`tick $nymd $nhms $JEDI_VARANAFRQ`) - endif - end + # generate IAU increment file + touch input.nml + $JEDI_MKIAU_MPIRUN -np $ncpus mkiau.x + + # clean up + #rm -f input.nml mkiau.rc + end endif -else - # this is one of those things ... the time in the file and the name-tag are inconsistent - # this is a consequence of the change from bin to nc4 agcm_import files - set hh = `echo $nhms | cut -c1-2` - if ( -e $EXPID.agcm_import_rst.${nymd}_${hh}00z.nc4 ) then - /bin/mv $EXPID.agcm_import_rst.${nymd}_${hh}00z.nc4 $EXPID.agcm_import_rst.${nymd0}_${hh0}00z.nc4 + +# if so, fake 4d + if ( ! -d iau ) mkdir iau + cd iau/ + if ( $JEDI_HYBRID ) then + set lst = `ls *agcm_import_rst.*nc4` + if ( $#lst == 1 ) then + set nowdate = ( $nymd0 $nhms0 ) + set enddate = (`tick $nowdate[1] $nowdate[2] $JEDI_VARWINDOW`) + set notdone = 1 + while ( $notdone ) + set nymd = $nowdate[1]; set nhms = $nowdate[2]; set hhmm = `echo $nhms | cut -c1-4` + /bin/cp $lst[1] $EXPID.agcm_import_rst.${nymd}_${hhmm}z.nc4 + if ( "$nowdate" == "$enddate" ) then + set notdone = 0 + else + set nowdate = (`tick $nymd $nhms $JEDI_VARANAFRQ`) + endif + end + endif else - echo " ${MYNAME}: cannot find: $EXPID.agcm_import_rst.${nymd}_${hh}00z.nc4, aborting ..." - exit (1) + # this is one of those things ... the time in the file and the name-tag are inconsistent + # this is a consequence of the change from bin to nc4 agcm_import files + set hh = `echo $nhms | cut -c1-2` + if ( -e $EXPID.agcm_import_rst.${nymd}_${hh}00z.nc4 ) then + /bin/mv $EXPID.agcm_import_rst.${nymd}_${hh}00z.nc4 $EXPID.agcm_import_rst.${nymd0}_${hh0}00z.nc4 + else + echo " ${MYNAME}: cannot find: $EXPID.agcm_import_rst.${nymd}_${hh}00z.nc4, aborting ..." + exit (1) + endif endif -endif -# arquive IAU increments: in 3D agcm_import stored as single file, in 4D as tar ball -# ---------------------- -cd $JEDIWORK/iau -tar cvf $FVWORK/$EXPID.jedi_agcmrst.${nymd0}_${hh0}z.tar $EXPID.agcm_import_rst* -cd - + + # arquive IAU increments: in 3D agcm_import stored as single file, in 4D as tar ball + # ---------------------- + cd $JEDIWORK/iau + tar cvf $FVWORK/$EXPID.jedi_agcmrst.${nymd0}_${hh0}z.tar $EXPID.agcm_import_rst* + cd - + +endif # central # arquive analysis # ---------------- cd $JEDIWORK/ana -tar cvf $FVWORK/$EXPID.jedi_ana.${nymd0}_${hh0}z.tar $EXPID.jedi_ana*.nc* +if ( $opt == "central" ) then + tar cvf $FVWORK/$EXPID.jedi_ana.${nymd0}_${hh0}z.tar $EXPID.jedi_ana*.nc* +else + if ( ! -d $FVWORK/updated_ens/$opt ) mkdir -p $FVWORK/updated_ens/$opt/ + foreach fn ( `ls $EXPID.jedi_ana.eta.*` ) + set sfx = `echo $fn | cut -d. -f3-` + /bin/mv $fn $FVWORK/updated_ens/$opt/$EXPID.ana.$sfx + end +endif cd - # arquive increments # ------------------ if ( -d $JEDIWORK/inc ) then cd $JEDIWORK/inc - tar cvf $FVWORK/$EXPID.jedi_inc.${nymd0}_${hh0}z.tar $EXPID.jedi_*.nc* + if ( $opt == "central" ) then + tar cvf $FVWORK/$EXPID.jedi_inc.${nymd0}_${hh0}z.tar $EXPID.jedi_*.nc* + else + if ( ! -d $FVWORK/updated_ens/$opt ) mkdir -p $FVWORK/updated_ens/$opt/ + foreach fn ( `ls $EXPID.jedi_inc.eta.*` ) + set sfx = `echo $fn | cut -d. -f3-` + /bin/mv $fn $FVWORK/updated_ens/$opt/$EXPID.inc.$sfx + end + endif cd - endif # If here, likely successful # -------------------------- -touch $FVWORK/.DONE_${MYNAME}.$yyyymmddhh +touch $FVWORK/.DONE_${MYNAME}.$opt.$yyyymmddhh echo " ${MYNAME}: Complete " exit(0) diff --git a/src/Applications/GEOSdas_App/jedi/jedi_run_atmens.csh b/src/Applications/GEOSdas_App/jedi/jedi_run_atmens.csh index 8efb2e28..23c1cf35 100755 --- a/src/Applications/GEOSdas_App/jedi/jedi_run_atmens.csh +++ b/src/Applications/GEOSdas_App/jedi/jedi_run_atmens.csh @@ -21,13 +21,17 @@ if ( $#argv < 5 ) then exit(1) endif -set nymda = $1 -set nhmsa = $2 +set nymda = $1 +set nhmsa = $2 set inpdir = $3 set outdir = $4 set jediwk = $5 -set hha = `echo $nhmsa | cut -c1-2` +set yyyya = `echo $nymda | cut -c1-4` +set mma = `echo $nymda | cut -c5-6` +set dda = `echo $nymda | cut -c7-8` +set hha = `echo $nhmsa | cut -c1-2` set yyyymmddhh = ${nymda}${hha} +setenv ISO_STATES_DATE "${yyyya}-${mma}-${dda}T${hha}:00:00Z" setenv SYSFAILED 0 if ( !($?LMOD_CMD) ) setenv SYSFAILED 1 @@ -39,6 +43,10 @@ if ( !($?FVHOME) ) setenv FAILED 1 if ( !($?FVWORK) ) setenv FAILED 1 if ( !($?SWELL_INSTALL) ) setenv FAILED 1 if ( !($?TIMEINC) ) setenv FAILED 1 +if ( !($?JEDI_ROOT) ) setenv FAILED 1 +if ( !($?JEDI_CRTM_COEFFS) ) setenv FAILED 1 + +if ( !($?NCSUFFIX) ) setenv NCSUFFIX nc4 if ( $FAILED ) then env @@ -49,11 +57,10 @@ endif if (-e $FVWORK/.DONE_${MYNAME}.$yyyymmddhh ) then echo " ${MYNAME}: already done" exit(0) +else + echo "${MYNAME}: Starting ... " endif -if ( !($?JEDI_ROOT) ) setenv FAILED 1 -if ( !($?JEDI_CRTM_COEFFS) ) setenv FAILED 1 - if ( $SYSFAILED ) then env echo " ${MYNAME}: not all SYSTEM required env vars defined" @@ -70,6 +77,8 @@ if ( ! $JEDI_RUN_EAANA ) then exit(0) endif +setenv JEDIBUILD $JEDI_ROOT + # set JEDI workdir setenv JEDIETC $FVHOME/run/jedi/Config setenv JEDIWORK $jediwk @@ -101,30 +110,33 @@ set mmi = `echo $nymdi | cut -c5-6` set ddi = `echo $nymdi | cut -c7-8` set hhi = `echo $nhmsi | cut -c1-2` - # get positioned - cd $JEDIWORK +# get positioned +cd $JEDIWORK + +# sanity check +if ( -e $JEDIETC/jedi_eda.yaml && \ + -e $JEDIETC/jedi_getkf.yaml ) then + echo "${MYNAME}: both EDA & GETKF are set to run ..." + echo "${MYNAME}: user should decide which, aborting ..." + exit(1) +endif +# prepare yaml to run JEDI-based atmos ens +set foundrc = 0 +if ( -e $JEDIETC/jedi_eda.yaml ) then + setenv SUITE eda_atmos + set myyaml = jedi_eda.yaml + set foundrc = 1 + echo "${MYNAME}: setting up to run JEDI-EDA" +endif +if ( -e $JEDIETC/jedi_getkf.yaml ) then + setenv SUITE localensembleda_2steps + set myyaml = jedi_getkf.yaml + set foundrc = 1 + echo "${MYNAME}: setting up to run JEDI-GETKF" +endif + +if ( ! -e $FVWORK/.DONE_${MYNAME}_${SUITE}.$yyyymmddhh ) then - # sanity check - if ( -e $JEDIETC/jedi_eda.yaml && \ - -e $JEDIETC/jedi_getkf.yaml ) then - echo "${MYNAME}: both EDA & GETKF are set to run ..." - echo "${MYNAME}: user should decide which, aborting ..." - exit(1) - endif - # prepare yaml to run JEDI-based atmos ens - set foundrc = 0 - if ( -e $JEDIETC/jedi_eda.yaml ) then - setenv SUITE eda_atmos - set myyaml = jedi_eda.yaml - set foundrc = 1 - echo "${MYNAME}: setting up to run JEDI-EDA" - endif - if ( -e $JEDIETC/jedi_getkf.yaml ) then - setenv SUITE localensembleda_2steps - set myyaml = jedi_getkf.yaml - set foundrc = 1 - echo "${MYNAME}: setting up to run JEDI-GETKF" - endif if ( $foundrc ) then vED -env $JEDIETC/$myyaml -o $myyaml else @@ -168,18 +180,110 @@ set hhi = `echo $nhmsi | cut -c1-2` echo "Trouble running JEDI atmos ensemble, aborting ..." exit 1 else -# if ( -d $JEDIWORK/${EXPID}-${SUITE}/run/${nymda}T${hha}0000Z/geos_atmosphere ) then -# if (! -d $outdir/ioda.${nymda}_${hha}0000 ) mkdir -p $outdir/ioda.${nymda}_${hha}0000 -# /bin/mv $JEDIWORK/${EXPID}-${SUITE}/run/${nymda}T${hha}0000Z/geos_atmosphere/*.nc4 \ -# $outdir/ioda.${nymda}_${hha}0000 -# /bin/mv $JEDIWORK/${EXPID}-${SUITE}/run/${nymda}T${hha}0000Z/geos_atmosphere/*.txt \ -# $outdir/ioda.${nymda}_${hha}0000 -# else -# echo "$JEDIWORK/${EXPID}-${SUITE}/run/${nymda}T${hha}0000Z/geos_atmosphere directory not found, aborting ..." -# exit 1 -# endif -# endif -#endif + touch $FVWORK/.DONE_${MYNAME}_${SUITE}.$yyyymmddhh + echo " ${MYNAME}: Complete " + endif + +endif # already run suite + +source $JEDIBUILD/modules + + # Rename all files from suite-name to expid; make sure there are not ana.eta (lat-lon) files present + cd $JEDIWORK/${EXPID}-${SUITE}/run/${nymda}T${hha}0000Z/geos_atmosphere/analysis/ + set members = `ls -d mem*` + foreach dir ( mean variance $members ) + cd $dir + /bin/rm *.ana.eta.*.nc4 + # rename swell-given filenames to experiment id name + foreach fn ( `ls ${SUITE}*nc4` ) + set sfx = `echo $fn | cut -d. -f2-` + /bin/mv $fn $EXPID.$sfx + end + cd - + end + + # mimic env configuration of central ADAS within each member directory + cd $FVWORK + /bin/ln -sf $FVHOME/atmens/central . + cd - + foreach mdir ( $members ) + set this = $JEDIWORK/${EXPID}-${SUITE}/run/${nymda}T${hha}0000Z/geos_atmosphere/analysis/$mdir + cd $this + /bin/ln -sf $FVHOME/fv3-jedi . + # reposition and rename analysis and increment files + mkdir ana inc + foreach dir ( ana inc ) + cd $dir + /bin/ln -sf ../*${dir}*nc4 . + cd - + end + # reposition cubed and lat-lon background files + set this = $JEDIWORK/${EXPID}-${SUITE}/run/${nymda}T${hha}0000Z/geos_atmosphere/analysis/$mdir/bkg + if ( ! -d $this ) mkdir $this + cd $this + foreach fn ( `ls $FVWORK/$mdir/${EXPID}.bkg.eta.*nc4` ) + /bin/ln -sf $fn . + /bin/ln -sf $JEDIWORK/${EXPID}-${SUITE}/run/${nymda}T${hha}0000Z/geos_atmosphere/ebkg/$mdir/geos*nc4 . + end + cd - + # create location of IAU increments + set this = $JEDIWORK/${EXPID}-${SUITE}/run/${nymda}T${hha}0000Z/geos_atmosphere/analysis/$mdir/iau + mkdir $this + end + + # Generate all lat-lon increments for each member (PARALLELIZATION TO BE DONE) + # the following assumes ensemble is running 3D scheme + foreach mdir ( $members ) + set this = $JEDIWORK/${EXPID}-${SUITE}/run/${nymda}T${hha}0000Z/geos_atmosphere/analysis/$mdir + cd $this + foreach cana (`ls ana/*ana.ceta*` ) + set yaml = diffstates_geos.yaml + if ( -e $JEDIETC/$yaml ) then + /bin/cp $JEDIETC/$yaml . + set ocres = `echorc.x -rc $yaml npx` # resolution wired in yaml + set ncres = `getgfiodim.x $cana | grep -v GFIO` # current ensemble analysis resolution + # reset wired resolution to ensemble resolution + if (-e sed_file ) rm -f sed_file + echo "s/$ocres/$ncres[1]/g" > sed_file + /bin/mv $yaml $yaml.hold + sed -f sed_file ./$yaml.hold > ./$yaml + /bin/rm $yaml.hold sed_file + + set cbkg = `ls bkg/geos*${nymda}_${hha}*nc4` + set bkgfn = `basename $cbkg` + set anafn = `basename $cana` + setenv JEDI_CUBED_BKG $bkgfn + setenv JEDI_CUBED_ANA $anafn + /bin/mv $yaml $yaml.hold + vED -env $yaml.hold -o $yaml + + $JEDI_GETINC_MPIRUN $JEDIBUILD/bin/fv3jedi_diffstates.x $yaml + if ($status) then + echo " ${MYNAME}: fail to run diffstates.x for member $mdir , aborting ... " + exit 1 + endif + /bin/mv *inc*.eta.*nc4 inc/ # JEDI diff does not place inc in subdir + else + echo " ${MYNAME}: missing Config/diffstates_geos.yaml file, aborting ... " + exit 1 + endif + end + end + + cd $JEDIWORK/${EXPID}-${SUITE}/run/${nymda}T${hha}0000Z/geos_atmosphere/ + set this = $FVWORK/updated_ens/$EXPID.atmens_olog.${nymda}_${hha}z.tar + if ( -e $this.gz ) then + /bin/rm $this.gz + endif + tar cvf $this *_mem*.log + gzip $this + +# The mean/variances at this point are calculated before recentering, so do not move them over ... +# ------------------------------------------------------------------------------------------------ +# /bin/mv $JEDIWORK/${EXPID}-${SUITE}/run/${nymda}T${hha}0000Z/geos_atmosphere/analysis/mean/*.nc4 \ +# $FVWORK/updated_ens/ensmean +# /bin/mv $JEDIWORK/${EXPID}-${SUITE}/run/${nymda}T${hha}0000Z/geos_atmosphere/analysis/variance/*.nc4 \ +# $FVWORK/updated_ens/ensrms touch $FVWORK/.DONE_${MYNAME}.$yyyymmddhh echo " ${MYNAME}: Complete " diff --git a/src/Applications/GEOSdas_App/jedi/jedi_swellset.csh b/src/Applications/GEOSdas_App/jedi/jedi_swellset.csh deleted file mode 100755 index 4fd66604..00000000 --- a/src/Applications/GEOSdas_App/jedi/jedi_swellset.csh +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/csh - -setenv MYNAME jedi_swellset.csh - -if ( ! $?FVHOME ) then - env - echo " ${MYNAME}: need FVHOME env" - exit (1) -endif - -if ( ! -d $FVHOME/run/jedi ) then - echo " ${MYNAME}: Nothing to do." -else - setenv JEDIDIR $FVHOME/run/jedi -endif - -if ( $#argv < 4 ) then - echo " ${MYNAME}: invalid arg list, aborting" - exit(1) -endif - -set nymda = $1 -set nhmsa = $2 -set jedidir = $3 -set jediwrk = $4 -set hha = `echo $nhmsa | cut -c1-2` -set yyyymmddhh = ${nymda}${hha} - -if (-e $FVWORK/.DONE_${MYNAME}.$yyyymmddhh ) then - echo " ${MYNAME}: already done" - exit(0) -endif - -setenv FAILED 0 -if ( !($?EXPID) ) setenv FAILED 1 -if ( !($?FVWORK) ) setenv FAILED 1 -if ( !($?JEDI_OBS_OPT) ) setenv FAILED 1 -if ( !($?JEDI_ROOT) ) setenv FAILED 1 - -if ( $FAILED ) then - env - echo " ${MYNAME}: not all required env vars defined" - exit (1) -endif - -if ( !($?JEDI_RUN_ADANA) ) setenv JEDI_RUN_ADANA 0 - -# Defaults -#if ( $JEDI_RUN_ADANA ) then -# source $FVHOME/run/jedi/JEDIadanaConfig.csh -#else -# source $FVHOME/run/jedi/JEDIanaConfig.csh -#endif - -source $JEDI_ROOT/modules -# This needs care: TBD wired for now -module unload py-pycodestyle/2.8.0 - -# Setup SWELL -# =========== -cd $jediwrk/swell -ln -s $jedidir/swell/* . - -if ( -e swell-geosadas-env.tmpl ) then - if ( -e swell-geosadas-env ) /bin/rm swell-geosadas-env - setenv YYYYMMDD_HHz ${nymda}_${hha}z - vED -env swell-geosadas-env.tmpl -o swell-geosadas-env -else - echo "${MYNAME}: swell-geosadas-env.tmpl not found, aborting ..." - exit (1) -endif -swell-geosadas-set_experiment.sh - -# Prep yaml for IODA converter -if ( -e swell-geosadas.yaml.tmpl ) then - if ( -e swell-geosadas.yaml ) /bin/rm swell-geosadas.yaml - vED -env swell-geosadas.yaml.tmpl -o swell-geosadas.yaml -else - echo "${MYNAME}: swell-geosadas.yaml.tmpl not found, aborting ..." - exit (1) -endif - -touch $FVWORK/.DONE_${MYNAME}.$yyyymmddhh -echo " ${MYNAME}: Complete " -exit(0) diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atm_ens.j b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atm_ens.j index b6f59843..2f8a54e7 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atm_ens.j +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atm_ens.j @@ -37,7 +37,6 @@ # Determines whether JEDI-based ensemble analysis will run # in place of GSI-based ensemble (for now, experiments # require a pure GSI-based initial cycle) - @ jedi_cycles = 0 setenv JEDI_CAN_RUN_NOW 0 if ( -d $FVHOME/run/jedi ) then set jedi_cycles = `ls $FVHOME/run/jedi/Config/geosvar.*z.yaml | wc` @@ -48,6 +47,7 @@ source $FVHOME/run/jedi/JEDIanaConfig.csh source $FVHOME/run/jedi/JEDIatmensConfig.csh setenv JEDI_CAN_RUN_NOW 1 + setenv ATMENS_ENSQRF 0 endif endif # tell which side of the machine we are on ... @@ -520,9 +520,11 @@ set amm = `echo ${anymd} | cut -c5-6` set ahh = `echo ${anhms} | cut -c1-2` cd $FVWORK/updated_ens -# if(! -e $HYBRIDGSI/${EXPID}.atmens_eana_brec.${nymdb}_${hhb}z.tar ) then -# tar -cvf $HYBRIDGSI/${EXPID}.atmens_eana_brec.${nymdb}_${hhb}z.tar mem0*/*.ana.eta*nc4 -# endif + if ( $ATMENS_SAVE_BREC ) then + if(! -e $HYBRIDGSI/${EXPID}.atmens_eana_brec.${nymdb}_${hhb}z.tar ) then + tar -cvf $HYBRIDGSI/${EXPID}.atmens_eana_brec.${nymdb}_${hhb}z.tar mem0*/*.ana.eta*nc4 + endif + endif cd - endif @@ -553,9 +555,11 @@ set amm = `echo ${anymd} | cut -c5-6` set ahh = `echo ${anhms} | cut -c1-2` cd $FVWORK/updated_ens -# if(! -e $HYBRIDGSI/${EXPID}.atmens_eana_arec.${nymdb}_${hhb}z.tar ) then -# tar -cvf $HYBRIDGSI/${EXPID}.atmens_eana_arec.${nymdb}_${hhb}z.tar mem0*/*.ana.eta*nc4 -# endif + if ( $ATMENS_SAVE_AREC ) then + if(! -e $HYBRIDGSI/${EXPID}.atmens_eana_arec.${nymdb}_${hhb}z.tar ) then + tar -cvf $HYBRIDGSI/${EXPID}.atmens_eana_arec.${nymdb}_${hhb}z.tar mem0*/*.ana.eta*nc4 + endif + endif cd - endif diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/AGCM.rc.tmpl b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/AGCM.rc.tmpl index db7732af..e8a8be13 100644 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/AGCM.rc.tmpl +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/AGCM.rc.tmpl @@ -7,7 +7,7 @@ IOSERVER_NODES: 1 NUM_BACKEND_PES: 0 DATAATM.GRID_TYPE: @GRID_TYPE -DATAATM.GRIDNAME: @AGCM_GRIDNAME +DATAATM.GRIDNAME: PE@AGCM_IMx@AGCM_JM-CF DATAATM.NF: @AGCM_NF DATAATM.LM: @AGCM_LM DATAATM.IM_WORLD: @AGCM_IM diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/AtmEnsConfig.csh b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/AtmEnsConfig.csh index ee7abd18..6a45e90a 100644 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/AtmEnsConfig.csh +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/AtmEnsConfig.csh @@ -22,6 +22,9 @@ unsetenv PMI_SIZE setenv ATMENS_BATCHSUB "sbatch" +setenv ATMENS_SAVE_BREC 0 # save analyses before recentering/inflation +setenv ATMENS_SAVE_AREC 0 # save analysis after recentering/inflation + # Turn online (variational) bias correction on by default setenv PREPQC 0 # The ensemble obsever never runs PREPQC (gets inputs from central instead) setenv ACFTBIAS @ACFTBIAS # 0: no aircraft bias correction diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/post_eana.csh b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/post_eana.csh index a73c4055..65388f0d 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/post_eana.csh +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/post_eana.csh @@ -86,6 +86,7 @@ if ( !($?FVWORK) ) setenv FAILED 1 if ( !($?NCSUFFIX) ) setenv NCSUFFIX nc4 if ( !($?ENSPARALLEL) ) setenv ENSPARALLEL 0 +if ( !($?ATMENS_ENSQRF) ) setenv ATMENS_ENSQRF 1 if ( !($?AENSADDINFLOC) ) then @@ -151,21 +152,45 @@ touch .no_archiving endif endif -# Calculate OmA's by running mean observer one final time -# ------------------------------------------------------- - obsvr_ensfinal.csh $expid $nymd $nhms $ENSWORK/updated_ens - if ( $status ) then - echo " ${MYNAME}: error running final observer, aborting ..." - exit(1) - endif +if ( $ATMENS_ENSQRF ) then -# Calculate obs impact on analysis -# -------------------------------- - atmens_obsimp0hr.csh $expid $nymd $nhms - if ( $status ) then - echo " ${MYNAME}: error running obs impact on analysis, aborting ..." - exit(1) - endif + # Calculate OmA's by running mean observer one final time + # ------------------------------------------------------- + obsvr_ensfinal.csh $expid $nymd $nhms $ENSWORK/updated_ens + if ( $status ) then + echo " ${MYNAME}: error running final observer, aborting ..." + exit(1) + endif + + # Calculate obs impact on analysis + # -------------------------------- + atmens_obsimp0hr.csh $expid $nymd $nhms + if ( $status ) then + echo " ${MYNAME}: error running obs impact on analysis, aborting ..." + exit(1) + endif + +else + + # TO-BE-DONE: this is a little out of place + # ln analysis to work ens dirs + # ----------------------------------------- + cd $FVWORK/updated_ens/mem001 + set lst = `ls *ana.eta.*nc4` + cd - + echo " ${MYNAME}: Linking member analysis to main location " + @ ic = 0 + while ( $ic < $nmem ) + @ ic = $ic + 1 + set memtag = `echo $ic |awk '{printf "%03d", $1}'` + cd $FVWORK/mem$memtag + foreach fn ($lst) + set pfx = `echo $fn | cut -d. -f1-4` + /bin/ln -s $FVWORK/updated_ens/mem$memtag/$fn $pfx.mem$memtag.nc4 + end + end + +endif # ATMENS_ENSQRF # Recenter ensemble # ----------------- From 75dda68777ee4a32bc2283b39721c7d4ad46ab26 Mon Sep 17 00:00:00 2001 From: Ricardo Todling Date: Tue, 13 Oct 2026 14:57:06 -0400 Subject: [PATCH 4/9] GSI 4 JEDI --- components.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components.yaml b/components.yaml index 0dcfe21f..00ee63d5 100644 --- a/components.yaml +++ b/components.yaml @@ -86,7 +86,7 @@ FMS: GEOSana_GridComp: local: ./src/Components/@GEOSana_GridComp remote: ../GEOSana_GridComp.git - tag: v5.46.0 + tag: v5.46.1 develop: develop mksi: From 57baf6adba94860ccd8db58c5216907322a28013 Mon Sep 17 00:00:00 2001 From: Ricardo Todling Date: Wed, 14 Oct 2026 09:22:35 -0400 Subject: [PATCH 5/9] still getting this to work --- .../GEOSdas_App/jedi/jedi_driver_atmens.csh | 9 +++++---- .../GEOSdas_App/jedi/jedi_run_atmens.csh | 18 +++++++++++------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/Applications/GEOSdas_App/jedi/jedi_driver_atmens.csh b/src/Applications/GEOSdas_App/jedi/jedi_driver_atmens.csh index 2e874994..39ec6915 100755 --- a/src/Applications/GEOSdas_App/jedi/jedi_driver_atmens.csh +++ b/src/Applications/GEOSdas_App/jedi/jedi_driver_atmens.csh @@ -143,11 +143,12 @@ endif # TBD: the following lines belong to # jedi_set # ---------------------------------- -cd $FVWORK - if ( ! -d mem001 ) then - /bin/ln -s $FVHOME/atmens/mem* . -endif +# if ( ! -d mem001 ) then +# /bin/ln -s $FVHOME/atmens/mem* . +#endif +cd $FVHOME/atmens set members = (`ls -d mem*`) +cd $FVWORK # Handle mean and variance output # ------------------------------- diff --git a/src/Applications/GEOSdas_App/jedi/jedi_run_atmens.csh b/src/Applications/GEOSdas_App/jedi/jedi_run_atmens.csh index 23c1cf35..4b9b0f5b 100755 --- a/src/Applications/GEOSdas_App/jedi/jedi_run_atmens.csh +++ b/src/Applications/GEOSdas_App/jedi/jedi_run_atmens.csh @@ -154,10 +154,14 @@ if ( ! -e $FVWORK/.DONE_${MYNAME}_${SUITE}.$yyyymmddhh ) then mkdir -p archive/$EXPID/atmens/Y$yyyyi/M$mmi cd $FVHOME/atmens mkdir $EXPID.atmens_ebkgx.${nymdi}_${hhi}z - mv ensbkgx $EXPID.atmens_ebkgx.${nymdi}_${hhi}z + if ( ! -d ensbkgx ) then + echo " ${MYNAME}: Trouble finding ensbkgx dir in FVHOME/atmens, aborting ..." + exit 1 + endif + /bin/mv ensbkgx $EXPID.atmens_ebkgx.${nymdi}_${hhi}z tar cvf $JEDIWORK/archive/$EXPID/atmens/Y$yyyyi/M$mmi/$EXPID.atmens_ebkgx.${nymdi}_${hhi}z.tar \ $EXPID.atmens_ebkgx.${nymdi}_${hhi}z - mv $EXPID.atmens_ebkgx.${nymdi}_${hhi}z/ensbkgx . + /bin/mv $EXPID.atmens_ebkgx.${nymdi}_${hhi}z/ensbkgx . /bin/rm -r $EXPID.atmens_ebkgx.${nymdi}_${hhi}z cd - endif @@ -170,14 +174,14 @@ if ( ! -e $FVWORK/.DONE_${MYNAME}_${SUITE}.$yyyymmddhh ) then endif $DRYRUN swell create ${SUITE} --skip-r2d2 --override $myyaml else - echo "Trouble finding $myyaml, aborting ..." + echo " ${MYNAME}: Trouble finding $myyaml, aborting ..." exit 1 endif # launch convert suite swell launch $JEDIWORK/${EXPID}-${SUITE}/${EXPID}-${SUITE}-suite -b -t PT30S if ( $status ) then - echo "Trouble running JEDI atmos ensemble, aborting ..." + echo " ${MYNAME}: Trouble running JEDI atmos ensemble, aborting ..." exit 1 else touch $FVWORK/.DONE_${MYNAME}_${SUITE}.$yyyymmddhh @@ -203,9 +207,9 @@ source $JEDIBUILD/modules end # mimic env configuration of central ADAS within each member directory - cd $FVWORK - /bin/ln -sf $FVHOME/atmens/central . - cd - +# cd $FVWORK +# /bin/ln -sf $FVHOME/atmens/central . +# cd - foreach mdir ( $members ) set this = $JEDIWORK/${EXPID}-${SUITE}/run/${nymda}T${hha}0000Z/geos_atmosphere/analysis/$mdir cd $this From fd15146f5bfce425a960fda3044fafe53bf7a0bd Mon Sep 17 00:00:00 2001 From: Ricardo Todling Date: Wed, 14 Oct 2026 09:23:58 -0400 Subject: [PATCH 6/9] minor --- .../NCEP_Etc/NCEP_enkf/scripts/gmao/atm_ens.j | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atm_ens.j b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atm_ens.j index 2f8a54e7..cf2916f2 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atm_ens.j +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atm_ens.j @@ -447,7 +447,21 @@ # Run observer for mean and each ensemble member # ---------------------------------------------- - if ( ! $JEDI_CAN_RUN_NOW ) then + if ( $JEDI_CAN_RUN_NOW ) then + cd $FVWORK/ + @ ic = 1 + while ( $ic <= $nmem ) + set memtag = `echo $ic |awk '{printf "%03d", $1}'` + if ( ! -d mem$memtag ) then + mkdir mem$memtag + cd mem$memtag + /bin/ln -sf $FVHOME/atmens/mem$memtag/$EXPID.bkg.eta.*.$NCSUFFIX . + cd - + endif + @ ic++ + end + cd - + else if ( $RUN_OBVSR || $DO_ATM_ENS ) then zeit_ci.x obsvr obsvr_ensemble.csh $OBSCLASS $EXPID $anymd $anhms |& tee -a atm_ens.log From 6ed186febcd0b7c82aa38aab0826475a6ebb42fc Mon Sep 17 00:00:00 2001 From: Ricardo Todling Date: Wed, 14 Oct 2026 14:50:11 -0400 Subject: [PATCH 7/9] JEDI ensemble now works --- .../NCEP_Etc/NCEP_enkf/scripts/gmao/etc/GEOS_ChemGridComp.rc | 1 + .../NCEP_Etc/NCEP_enkf/scripts/gmao/gcm_ensset_rc.csh | 1 + 2 files changed, 2 insertions(+) diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/GEOS_ChemGridComp.rc b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/GEOS_ChemGridComp.rc index e18df797..5f6205ba 100644 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/GEOS_ChemGridComp.rc +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/GEOS_ChemGridComp.rc @@ -24,6 +24,7 @@ ENABLE_TR: .TRUE. ENABLE_DNA: .FALSE. ENABLE_HEMCO: .TRUE. + ENABLE_RRG: .FALSE. # Use only for diagnostic timing tests: # -------------------------------------------------- diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/gcm_ensset_rc.csh b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/gcm_ensset_rc.csh index a1d8bcc4..24396a12 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/gcm_ensset_rc.csh +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/gcm_ensset_rc.csh @@ -229,6 +229,7 @@ cd $ENSWORK/$member # -------------------------------------------------------------------------------- /bin/ln -sf $ENSWORK/ExtData $ENSWORK/${member}/ /bin/cp $ATMENSETC/GEOS_ChemGridComp.rc . + /bin/cp $FVHOME/run/GWD_GridComp.rc . /bin/cp $FVHOME/run/gocart/*.rc . /bin/cp $FVHOME/run/gocart/*.yaml . if(-e ExtData.rc) /bin/rm -f ExtData.rc From d7cfa004e7e5ecbcd330036eab1143ac6a05eaec Mon Sep 17 00:00:00 2001 From: Ricardo Todling Date: Wed, 14 Oct 2026 15:03:05 -0400 Subject: [PATCH 8/9] fix kx odsmeta --- components.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components.yaml b/components.yaml index 00ee63d5..eae3ff04 100644 --- a/components.yaml +++ b/components.yaml @@ -35,7 +35,7 @@ NCEP_Shared: GMAO_Shared: local: ./src/Shared/@GMAO_Shared remote: ../GMAO_Shared.git - tag: g5.45.1 + tag: g5.45.2 sparse: ./config/GMAO_Shared.sparse develop: main From 2621218df2d2baa12454b3dcde78f904db0ff0a8 Mon Sep 17 00:00:00 2001 From: Ricardo Todling Date: Wed, 14 Oct 2026 17:47:44 -0400 Subject: [PATCH 9/9] as in j0055 --- .../GEOSdas_App/jedi/etc/JEDIatmensConfig.csh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Applications/GEOSdas_App/jedi/etc/JEDIatmensConfig.csh b/src/Applications/GEOSdas_App/jedi/etc/JEDIatmensConfig.csh index 5355e526..d6849dad 100644 --- a/src/Applications/GEOSdas_App/jedi/etc/JEDIatmensConfig.csh +++ b/src/Applications/GEOSdas_App/jedi/etc/JEDIatmensConfig.csh @@ -3,9 +3,10 @@ # SLURM specials # Top options -setenv JEDI_RUN_EAANA 1 # run ensemble analysis -setenv JEDI_MKIAU 1 # calculates IAU output -setenv JEDI_MKIAU_CUBED 1 # 0=use lat-lon ana/bkg; 1=use cubed ana/bkg (no remap) +setenv JEDI_RUN_ANA 0 # make sure turn off central ADAS analysis ... +setenv JEDI_RUN_EAANA 1 # and turn on running of ensemble analysis +#setenv JEDI_MKIAU 1 # calculates IAU output +#setenv JEDI_MKIAU_CUBED 0 # 0=use lat-lon ana/bkg; 1=use cubed ana/bkg (no remap) # Details ... setenv JEDI_OBS_OPT 1 # 1= point to xexp-like set (data in tar-balls; data from existing exp) @@ -15,5 +16,4 @@ setenv JEDI_OBS_OPT 1 # 1= point to xexp-like set (data in tar-balls; d setenv SWELL_INSTALL /discover/nobackup/projects/gmao/advda/rtodling/JEDI1/2026/SWELL/Jul/ setenv JEDI_ROOT /discover/nobackup/projects/gmao/advda/swell/JediBundles/fv3_soca_SLES15_08142026/build-intel-release/ -setenv JEDI_RUN_ANA 0