// Purpose: CDL to generate large netCDF file for testing Large File Support (LFS)

// Usage:
// Create big.nc from big.cdl on a beefy 64-bit computer
// The file creation command fails on systems with insufficient RAM :)
// Transfer big.nc to a 32-bit system as an LFS testfile

// ncgen -b -o big.nc big.cdl
// ncgen -b -o ~/nco/data/big.nc ~/nco/data/big.cdl
// scp ~/nco/data/big.cdl esmf.ess.uci.edu:nco/data
// scp ~/nco/data/big.cdl dust.ess.uci.edu:nco/data

// Test 64-bit netCDF/NCO capabilities
// 32-bit machines do not work with files exceeding ~2 GB unless LFS enabled
// One billion floats are nco_typ_lng(NC_FLOAT)*10^9 = sizeof(float)*10^9 = 4*10^9 B = 4 GB

// Create file with variable at least 4 GB in size
// Expect failure on "small RAM" (< ~4 GB) machines:
// ncap2 -5 -D 3 -O -s "wvl_1e0[wvl_1e0]=1.0f" -s "wvl_1e1[wvl_1e1]=1.0f" -s "wvl_1e2[wvl_1e2]=1.0f" -s "wvl_1e3[wvl_1e3]=1.0f" -s "wvl_1e4[wvl_1e4]=1.0f" -s "wvl_1e5[wvl_1e5]=1.0f" -s "wvl_1e6[wvl_1e6]=1.0f" -s "wvl_1e7[wvl_1e7]=1.0f" -s "wvl_1e8[wvl_1e8]=1.0f" -s "wvl_1e9[wvl_1e9]=1.0f" ~/nco/data/big.nc ~/foo_big.nc
// ncap2 -5 -D 3 -O -s "wvl_1e0[wvl_1e0]=1.0d" -s "wvl_1e1[wvl_1e1]=1.0d" -s "wvl_1e2[wvl_1e2]=1.0d" -s "wvl_1e3[wvl_1e3]=1.0d" -s "wvl_1e4[wvl_1e4]=1.0d" -s "wvl_1e5[wvl_1e5]=1.0d" -s "wvl_1e6[wvl_1e6]=1.0d" -s "wvl_1e7[wvl_1e7]=1.0d" -s "wvl_1e8[wvl_1e8]=1.0d" -s "wvl_1e9[wvl_1e9]=1.0d" ~/nco/data/big.nc ~/foo_big.nc
// ncap2 -4 --cnk_dmn=wvl_1e9,100000 -D 3 -O -s "wvl_1e0[wvl_1e0]=1.0d" -s "wvl_1e1[wvl_1e1]=1.0d" -s "wvl_1e2[wvl_1e2]=1.0d" -s "wvl_1e3[wvl_1e3]=1.0d" -s "wvl_1e4[wvl_1e4]=1.0d" -s "wvl_1e5[wvl_1e5]=1.0d" -s "wvl_1e6[wvl_1e6]=1.0d" -s "wvl_1e7[wvl_1e7]=1.0d" -s "wvl_1e8[wvl_1e8]=1.0d" -s "wvl_1e9[wvl_1e9]=1.0d" ~/nco/data/big.nc ~/foo_big.nc
// ls -l ~/foo_big.nc

// On small-RAM machines, create 4 GB file with multiple variables instead
// This reduces peak memory usage to ~400 MB, and still tests LFS capability
// ncap2 -5 -D 3 -O -s "wvl_1e8[wvl_1e8]=1.0f" ~/nco/data/big.nc ${HOME}/big.nc # Data buffer size: 400 MB
// ncrename -D 3 -d wvl_1e8,wvl ${HOME}/big.nc # Rename because ncecat does not concatenate coordinate variables
// Aggregate ten files together
// ncecat -u time -D 3 -O -p ${HOME} big.nc big.nc big.nc big.nc big.nc big.nc big.nc big.nc big.nc big.nc ${HOME}/big.nc 
// ls -l ${HOME}/big.nc

// Test data access:
// This is where LFS is useful
// These commands should work on 32-bit systems with LFS support even though files exceed 2 GB
// ncks -m -M -H ${HOME}/big.nc | m
// ncks -H -d wvl_1e9,999999999 ${HOME}/big.nc | m

// Valid CDF/netCDF files need not have any defined variable or data
// Use ncap LHS-casting to define variables with big dimensions
netcdf big {
dimensions:
// CDL file only used to supply dimension size = one billion to ncap 
// Create actual variable with ncap LHS-casting
	wvl_1e9=1000000000,
	wvl_1e8=100000000,
	wvl_1e7=10000000,
	wvl_1e6=1000000,
	wvl_1e5=100000,
	wvl_1e4=10000,
	wvl_1e3=1000,
	wvl_1e2=100,
	wvl_1e1=10,
	wvl_1e0=1,
	lev=32,
	lat=128,
	lon=256,
	time=unlimited;
 variables:
//	float wvl_1e9(wvl_1e9);
//	float wvl_1e8(wvl_1e8);
//	float wvl_1e7(wvl_1e7);
//	float wvl_1e6(wvl_1e6);
//	float wvl_1e5(wvl_1e5);
//	float wvl_1e4(wvl_1e4);
//	float wvl_1e3(wvl_1e3);
//	float wvl_1e2(wvl_1e2);
//	float wvl_1e1(wvl_1e1);
//	float wvl_1e0(wvl_1e0);
	double time(time);
	time:long_name = "Time coordinate record variable";
data:
//	wvl_1e9=1;
//	wvl_1e8=1;
//	wvl_1e7=1;
//	wvl_1e6=1;
//	wvl_1e5=1;
//	wvl_1e4=1;
//	wvl_1e3=1;
//	wvl_1e2=1;
//	wvl_1e1=1;
//	wvl_1e0=1;
	time=1.,2.,3.,4.,5.,6.,7.,8.,9.,10.;
}
