Quick start

import xbitinfo as xb

import xarray as xr

/home/docs/checkouts/readthedocs.org/user_builds/xbitinfo/conda/latest/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html
  from .autonotebook import tqdm as notebook_tqdm

ds = xr.tutorial.load_dataset("eraint_uvz").astype("float32")

xb.plot_distribution(ds)
ds

/home/docs/checkouts/readthedocs.org/user_builds/xbitinfo/conda/latest/lib/python3.12/site-packages/xarray/conventions.py:286: SerializationWarning: variable 'z' has non-conforming '_FillValue' nan defined, dropping '_FillValue' entirely.
  var = coder.decode(var, name=name)
/home/docs/checkouts/readthedocs.org/user_builds/xbitinfo/conda/latest/lib/python3.12/site-packages/xarray/conventions.py:286: SerializationWarning: variable 'u' has non-conforming '_FillValue' nan defined, dropping '_FillValue' entirely.
  var = coder.decode(var, name=name)
/home/docs/checkouts/readthedocs.org/user_builds/xbitinfo/conda/latest/lib/python3.12/site-packages/xarray/conventions.py:286: SerializationWarning: variable 'v' has non-conforming '_FillValue' nan defined, dropping '_FillValue' entirely.
  var = coder.decode(var, name=name)

<xarray.Dataset> Size: 8MB
Dimensions:    (month: 2, level: 3, latitude: 241, longitude: 480)
Coordinates:
  * longitude  (longitude) float32 2kB -180.0 -179.2 -178.5 ... 178.5 179.2
  * latitude   (latitude) float32 964B 90.0 89.25 88.5 ... -88.5 -89.25 -90.0
  * level      (level) int32 12B 200 500 850
  * month      (month) int32 8B 1 7
Data variables:
    z          (month, level, latitude, longitude) float32 3MB 1.068e+05 ... ...
    u          (month, level, latitude, longitude) float32 3MB 1.282 ... 3.539
    v          (month, level, latitude, longitude) float32 3MB -0.04676 ... 3...
Attributes:
    Conventions:  CF-1.0
    Info:         Monthly ERA-Interim data. Downloaded and edited by fabien.m...

xarray.Dataset

• Dimensions:
  • month: 2
  • level: 3
  • latitude: 241
  • longitude: 480
• Coordinates: (4)
  • longitude
    (longitude)
    float32
    -180.0 -179.2 ... 178.5 179.2

    units :

    degrees_east

    long_name :

    longitude

    array([-180.  , -179.25, -178.5 , ...,  177.75,  178.5 ,  179.25],
          dtype=float32)

  • latitude
    (latitude)
    float32
    90.0 89.25 88.5 ... -89.25 -90.0

    units :

    degrees_north

    long_name :

    latitude

    array([ 90.  ,  89.25,  88.5 , ..., -88.5 , -89.25, -90.  ], dtype=float32)

  • level
    (level)
    int32
    200 500 850

    units :

    millibars

    long_name :

    pressure_level

    array([200, 500, 850], dtype=int32)

  • month
    (month)
    int32
    1 7

    array([1, 7], dtype=int32)

• Data variables: (3)
  • z
    (month, level, latitude, longitude)
    float32
    1.068e+05 1.068e+05 ... 1.178e+04

    number_of_significant_digits :

    5

    units :

    m**2 s**-2

    long_name :

    Geopotential

    standard_name :

    geopotential

    array([[[[106837.516, 106839.234, 106837.516, ..., 106839.234,
              106837.516, 106839.234],
             [106856.484, 106858.21 , 106856.484, ..., 106856.484,
              106856.484, 106856.484],
             [106884.086, 106884.086, 106884.086, ..., 106884.086,
              106884.086, 106884.086],
             ...,
             [109795.94 , 109795.94 , 109795.94 , ..., 109795.94 ,
              109795.94 , 109795.94 ],
             [109802.836, 109804.56 , 109804.56 , ..., 109804.56 ,
              109802.836, 109802.836],
             [109808.01 , 109808.01 , 109808.01 , ..., 109808.01 ,
              109808.01 , 109808.01 ]],
    
            [[ 49723.58 ,  49723.58 ,  49723.58 , ...,  49723.58 ,
               49723.58 ,  49723.58 ],
             [ 49744.277,  49744.277,  49744.277, ...,  49746.004,
               49746.004,  49746.004],
             [ 49771.88 ,  49771.88 ,  49771.88 , ...,  49771.88 ,
               49771.88 ,  49771.88 ],
    ...
             [ 47891.598,  47889.875,  47889.875, ...,  47893.324,
               47891.598,  47891.598],
             [ 47941.625,  47941.625,  47941.625, ...,  47943.348,
               47941.625,  47941.625],
             [ 47974.4  ,  47974.4  ,  47974.4  , ...,  47974.4  ,
               47974.4  ,  47974.4  ]],
    
            [[ 13485.926,  13485.926,  13485.926, ...,  13485.926,
               13485.926,  13485.926],
             [ 13484.2  ,  13484.2  ,  13484.2  , ...,  13484.2  ,
               13484.2  ,  13484.2  ],
             [ 13484.2  ,  13484.2  ,  13484.2  , ...,  13482.476,
               13482.476,  13484.2  ],
             ...,
             [ 11360.691,  11358.967,  11357.242, ...,  11364.142,
               11362.417,  11362.417],
             [ 11612.546,  11612.546,  11612.546, ...,  11612.546,
               11612.546,  11612.546],
             [ 11776.424,  11776.424,  11776.424, ...,  11776.424,
               11776.424,  11776.424]]]], dtype=float32)

  • u
    (month, level, latitude, longitude)
    float32
    1.282 1.282 1.282 ... 3.578 3.539

    number_of_significant_digits :

    2

    units :

    m s**-1

    long_name :

    U component of wind

    standard_name :

    eastward_wind

    array([[[[ 1.2817602e+00,  1.2817602e+00,  1.2817602e+00, ...,
               1.2817602e+00,  1.2817602e+00,  1.2817602e+00],
             [ 2.0162134e+00,  2.0319405e+00,  2.0240769e+00, ...,
               2.0083499e+00,  2.0083499e+00,  2.0240769e+00],
             [ 2.8198657e+00,  2.8355927e+00,  2.8355927e+00, ...,
               2.7962751e+00,  2.7962751e+00,  2.8198657e+00],
             ...,
             [-5.0797796e-01, -4.9225092e-01, -4.8438740e-01, ...,
              -5.4729557e-01, -5.3156853e-01, -5.2370501e-01],
             [-5.7088619e-01, -5.6302267e-01, -5.4729557e-01, ...,
              -5.9447676e-01, -5.8661324e-01, -5.7874972e-01],
             [-5.3943205e-01, -5.3943205e-01, -5.3156853e-01, ...,
              -5.5515909e-01, -5.4729557e-01, -5.4729557e-01]],
    
            [[ 1.9218512e+00,  1.9061241e+00,  1.8903971e+00, ...,
               1.9611688e+00,  1.9533052e+00,  1.9375782e+00],
             [ 2.3040185e+00,  2.3040185e+00,  2.2820005e+00, ...,
               2.3354726e+00,  2.3197455e+00,  2.3197455e+00],
             [ 2.7428031e+00,  2.7428031e+00,  2.7349396e+00, ...,
               2.7663937e+00,  2.7506666e+00,  2.7585301e+00],
    ...
              -4.4680490e+00, -4.4680490e+00, -4.4995031e+00],
             [-3.5543075e+00, -3.5857615e+00, -3.5936251e+00, ...,
              -3.4929719e+00, -3.5071263e+00, -3.5385804e+00],
             [-2.3905058e+00, -2.4219599e+00, -2.4376869e+00, ...,
              -2.3197341e+00, -2.3354611e+00, -2.3669152e+00]],
    
            [[-1.6355559e-01, -1.5569207e-01, -1.5569207e-01, ...,
              -1.8714617e-01, -1.7928264e-01, -1.7141911e-01],
             [ 7.8692473e-03,  1.5732773e-02,  2.3596296e-02, ...,
              -1.5721327e-02, -7.8578023e-03,  5.7223951e-06],
             [ 2.7365637e-01,  2.8151992e-01,  2.8938344e-01, ...,
               2.4220228e-01,  2.5006580e-01,  2.6579285e-01],
             ...,
             [-1.6875067e+00, -1.6875067e+00, -1.6953702e+00, ...,
              -1.6639161e+00, -1.6717796e+00, -1.6796432e+00],
             [ 1.4531851e+00,  1.4295945e+00,  1.3902769e+00, ...,
               1.5554109e+00,  1.5239568e+00,  1.4925027e+00],
             [ 3.4851198e+00,  3.4536657e+00,  3.3986211e+00, ...,
               3.6250906e+00,  3.5779095e+00,  3.5385919e+00]]]],
          dtype=float32)

  • v
    (month, level, latitude, longitude)
    float32
    -0.04676 -0.06253 ... 3.328 3.383

    number_of_significant_digits :

    2

    units :

    m s**-1

    long_name :

    V component of wind

    standard_name :

    northward_wind

    array([[[[-0.04675769, -0.06252575, -0.07829381, ...,  0.00771379,
              -0.00757645, -0.02334451],
             [ 0.14054775,  0.10948945,  0.08607627, ...,  0.22655535,
               0.19549705,  0.17208387],
             [ 0.33597612,  0.2967949 ,  0.25761366, ...,  0.45304203,
               0.4138608 ,  0.37515736],
             ...,
             [-0.6875143 , -0.70328236, -0.7185726 , ..., -0.656456  ,
              -0.6641011 , -0.6798692 ],
             [-0.5154991 , -0.53126717, -0.5389123 , ..., -0.49208593,
              -0.50020885, -0.507854  ],
             [-0.37502003, -0.38266516, -0.39078808, ..., -0.35160685,
              -0.35925198, -0.3673749 ]],
    
            [[-1.0778933 , -1.1094294 , -1.1328425 , ..., -1.0076537 ,
              -1.0310669 , -1.062603  ],
             [-0.92976904, -0.96082735, -0.99236345, ..., -0.8279934 ,
              -0.8595295 , -0.89823294],
             [-0.765399  , -0.81270313, -0.8514066 , ..., -0.6483331 ,
              -0.6875143 , -0.72669554],
    ...
             [ 1.5783081 ,  1.5075908 ,  1.4373512 , ...,  1.7813816 ,
               1.7187872 ,  1.6485476 ],
             [ 1.8358531 ,  1.7890267 ,  1.7345552 , ...,  1.9844551 ,
               1.9376287 ,  1.8908024 ],
             [ 1.9218607 ,  1.8908024 ,  1.8592663 , ...,  2.0155134 ,
               1.9844551 ,  1.952919  ]],
    
            [[ 0.51563644,  0.51563644,  0.51563644, ...,  0.5079913 ,
               0.5079913 ,  0.5079913 ],
             [ 0.54669476,  0.54669476,  0.54669476, ...,  0.5390496 ,
               0.5390496 ,  0.54669476],
             [ 0.585876  ,  0.585876  ,  0.585876  , ...,  0.585876  ,
               0.57823086,  0.585876  ],
             ...,
             [ 4.734309  ,  4.7500772 ,  4.7500772 , ...,  4.734309  ,
               4.718541  ,  4.734309  ],
             [ 3.195251  ,  3.226787  ,  3.2420774 , ...,  3.1250114 ,
               3.1484246 ,  3.179483  ],
             [ 3.4217377 ,  3.468564  ,  3.5077453 , ...,  3.2889037 ,
               3.328085  ,  3.3830342 ]]]], dtype=float32)

• Indexes: (4)
  • longitude
    PandasIndex

    PandasIndex(Index([ -180.0, -179.25,  -178.5, -177.75,  -177.0, -176.25,  -175.5, -174.75,
            -174.0, -173.25,
           ...
             172.5,  173.25,   174.0,  174.75,   175.5,  176.25,   177.0,  177.75,
             178.5,  179.25],
          dtype='float32', name='longitude', length=480))

  • latitude
    PandasIndex

    PandasIndex(Index([  90.0,  89.25,   88.5,  87.75,   87.0,  86.25,   85.5,  84.75,   84.0,
            83.25,
           ...
           -83.25,  -84.0, -84.75,  -85.5, -86.25,  -87.0, -87.75,  -88.5, -89.25,
            -90.0],
          dtype='float32', name='latitude', length=241))

  • level
    PandasIndex

    PandasIndex(Index([200, 500, 850], dtype='int32', name='level'))

  • month
    PandasIndex

    PandasIndex(Index([1, 7], dtype='int32', name='month'))

• Attributes: (2)

  Conventions :

  CF-1.0

  Info :

  Monthly ERA-Interim data. Downloaded and edited by fabien.maussion@uibk.ac.at

NOTE: If you plan to use the example datasets provided by xarray, you will need to install the pooch package separately using the following command:

pip install pooch

Requirement already satisfied: pooch in /home/docs/checkouts/readthedocs.org/user_builds/xbitinfo/conda/latest/lib/python3.12/site-packages (1.8.1)

Requirement already satisfied: platformdirs>=2.5.0 in /home/docs/checkouts/readthedocs.org/user_builds/xbitinfo/conda/latest/lib/python3.12/site-packages (from pooch) (4.2.0)
Requirement already satisfied: packaging>=20.0 in /home/docs/checkouts/readthedocs.org/user_builds/xbitinfo/conda/latest/lib/python3.12/site-packages (from pooch) (24.0)
Requirement already satisfied: requests>=2.19.0 in /home/docs/checkouts/readthedocs.org/user_builds/xbitinfo/conda/latest/lib/python3.12/site-packages (from pooch) (2.31.0)
Requirement already satisfied: charset-normalizer<4,>=2 in /home/docs/checkouts/readthedocs.org/user_builds/xbitinfo/conda/latest/lib/python3.12/site-packages (from requests>=2.19.0->pooch) (3.3.2)
Requirement already satisfied: idna<4,>=2.5 in /home/docs/checkouts/readthedocs.org/user_builds/xbitinfo/conda/latest/lib/python3.12/site-packages (from requests>=2.19.0->pooch) (3.7)
Requirement already satisfied: urllib3<3,>=1.21.1 in /home/docs/checkouts/readthedocs.org/user_builds/xbitinfo/conda/latest/lib/python3.12/site-packages (from requests>=2.19.0->pooch) (1.26.18)
Requirement already satisfied: certifi>=2017.4.17 in /home/docs/checkouts/readthedocs.org/user_builds/xbitinfo/conda/latest/lib/python3.12/site-packages (from requests>=2.19.0->pooch) (2024.2.2)

Note: you may need to restart the kernel to use updated packages.

Without installing pooch, you will not be able to download and load the example datasets, which may result in errors or unexpected behavior.

Get information content per bit

using xbitinfo.xbitinfo.get_bitinformation()

info_per_bit = xb.get_bitinformation(ds, dim="longitude", implementation="python")

info_per_bit

  0%|          | 0/3 [00:00<?, ?it/s]

Processing var: z for dim: longitude:   0%|          | 0/3 [00:00<?, ?it/s]

Processing var: z for dim: longitude:  33%|███▎      | 1/3 [00:01<00:02,  1.23s/it]

Processing var: u for dim: longitude:  33%|███▎      | 1/3 [00:01<00:02,  1.23s/it]

Processing var: u for dim: longitude:  67%|██████▋   | 2/3 [00:02<00:01,  1.15s/it]

Processing var: v for dim: longitude:  67%|██████▋   | 2/3 [00:02<00:01,  1.15s/it]

Processing var: v for dim: longitude: 100%|██████████| 3/3 [00:03<00:00,  1.13s/it]

Processing var: v for dim: longitude: 100%|██████████| 3/3 [00:03<00:00,  1.14s/it]

<xarray.Dataset> Size: 1kB
Dimensions:     (bitfloat32: 32)
Coordinates:
  * bitfloat32  (bitfloat32) <U3 384B '±' 'e1' 'e2' 'e3' ... 'm21' 'm22' 'm23'
    dim         <U9 36B 'longitude'
Data variables:
    z           (bitfloat32) float64 256B 0.0 0.0 0.0 ... 0.005199 0.007699
    u           (bitfloat32) float64 256B 0.7816 0.4274 0.0 ... 0.01148 0.1475
    v           (bitfloat32) float64 256B 0.8752 0.7756 0.0 ... 0.06165 0.05304
Attributes:
    xbitinfo_description:       bitinformation calculated by xbitinfo.get_bit...
    python_repository:          https://github.com/observingClouds/xbitinfo
    julia_repository:           https://github.com/milankl/BitInformation.jl
    reference_paper:            http://www.nature.com/articles/s43588-021-001...
    xbitinfo_version:           0.1.dev193+g27c240d.d20240417
    BitInformation.jl_version:  implementation='python'

xarray.Dataset

• Dimensions:
  • bitfloat32: 32
• Coordinates: (2)
  • bitfloat32
    (bitfloat32)
    <U3
    '±' 'e1' 'e2' ... 'm21' 'm22' 'm23'

    description :

    name of the bits: '±' refers to the sign bit, 'e' to the exponents bits and 'm' to the mantissa bits.

    array(['±', 'e1', 'e2', 'e3', 'e4', 'e5', 'e6', 'e7', 'e8', 'm1', 'm2', 'm3',
           'm4', 'm5', 'm6', 'm7', 'm8', 'm9', 'm10', 'm11', 'm12', 'm13', 'm14',
           'm15', 'm16', 'm17', 'm18', 'm19', 'm20', 'm21', 'm22', 'm23'],
          dtype='<U3')

  • dim
    ()
    <U9
    'longitude'

    description :

    dimension of the source dataset along which the bitwise information has been analysed.

    array('longitude', dtype='<U9')

• Data variables: (3)
  • z
    (bitfloat32)
    float64
    0.0 0.0 0.0 ... 0.005199 0.007699

    long_name :

    z bitwise information

    units :

    1

    source_number_of_significant_digits :

    5

    source_units :

    m**2 s**-2

    source_long_name :

    Geopotential

    source_standard_name :

    geopotential

    array([0.        , 0.        , 0.        , 0.        , 0.91829583,
           0.91829583, 0.91829583, 0.91829583, 0.91829583, 0.40596261,
           0.97813287, 0.92341985, 0.94109422, 0.90208436, 0.82484959,
           0.71155894, 0.54404085, 0.34755517, 0.18092331, 0.07436806,
           0.02217038, 0.01245641, 0.00377219, 0.00628822, 0.01224325,
           0.00389673, 0.01004858, 0.0131198 , 0.00564089, 0.00887582,
           0.0051986 , 0.00769888])

  • u
    (bitfloat32)
    float64
    0.7816 0.4274 ... 0.01148 0.1475

    long_name :

    u bitwise information

    units :

    1

    source_number_of_significant_digits :

    2

    source_units :

    m s**-1

    source_long_name :

    U component of wind

    source_standard_name :

    eastward_wind

    array([7.81646180e-01, 4.27436423e-01, 0.00000000e+00, 0.00000000e+00,
           1.71227095e-04, 0.00000000e+00, 6.35104053e-01, 8.05002303e-01,
           6.78224433e-01, 5.35060838e-01, 3.99149342e-01, 2.46776345e-01,
           1.16042153e-01, 3.33513654e-02, 5.73779013e-03, 3.48614977e-03,
           5.59982312e-03, 9.19817640e-03, 7.71894512e-03, 5.38088606e-03,
           2.65337337e-03, 6.27602119e-04, 3.53690832e-04, 4.95650149e-04,
           3.34290451e-03, 2.51371722e-03, 2.11808385e-03, 1.17905219e-03,
           1.13182091e-03, 3.28875231e-03, 1.14760822e-02, 1.47515177e-01])

  • v
    (bitfloat32)
    float64
    0.8752 0.7756 ... 0.06165 0.05304

    long_name :

    v bitwise information

    units :

    1

    source_number_of_significant_digits :

    2

    source_units :

    m s**-1

    source_long_name :

    V component of wind

    source_standard_name :

    northward_wind

    array([8.75249762e-01, 7.75560757e-01, 0.00000000e+00, 0.00000000e+00,
           0.00000000e+00, 2.74549688e-03, 1.17783327e-01, 5.01420134e-01,
           4.06812530e-01, 2.29018531e-01, 1.01658673e-01, 2.73218922e-02,
           3.76688041e-03, 4.52212736e-04, 1.53181859e-04, 1.30129511e-04,
           5.09324245e-05, 4.18079116e-05, 3.43536304e-05, 2.87844951e-05,
           3.84562184e-05, 1.71014710e-04, 7.26738394e-04, 4.54372431e-04,
           6.11464268e-04, 5.30345968e-04, 2.06533122e-04, 7.77135473e-04,
           6.67379672e-03, 1.99001671e-02, 6.16532925e-02, 5.30433184e-02])

• Indexes: (1)
  • bitfloat32
    PandasIndex

    PandasIndex(Index(['±', 'e1', 'e2', 'e3', 'e4', 'e5', 'e6', 'e7', 'e8', 'm1', 'm2', 'm3',
           'm4', 'm5', 'm6', 'm7', 'm8', 'm9', 'm10', 'm11', 'm12', 'm13', 'm14',
           'm15', 'm16', 'm17', 'm18', 'm19', 'm20', 'm21', 'm22', 'm23'],
          dtype='object', name='bitfloat32'))

• Attributes: (6)

  xbitinfo_description :

  bitinformation calculated by xbitinfo.get_bitinformation wrapping bitinformation.jl

  python_repository :

  https://github.com/observingClouds/xbitinfo

  julia_repository :

  https://github.com/milankl/BitInformation.jl

  reference_paper :

  http://www.nature.com/articles/s43588-021-00156-2

  xbitinfo_version :

  0.1.dev193+g27c240d.d20240417

  BitInformation.jl_version :

  implementation='python'

Visualize information content

using xbitinfo.graphics.plot_bitinformation()

fig = xb.plot_bitinformation(info_per_bit)

Get keepbits

using xbitinfo.xbitinfo.get_keepbits()

keepbits = xb.get_keepbits(info_per_bit, 0.99)
keepbits

<xarray.Dataset> Size: 68B
Dimensions:   (inflevel: 1)
Coordinates:
    dim       <U9 36B 'longitude'
  * inflevel  (inflevel) float64 8B 0.99
Data variables:
    z         (inflevel) int64 8B 10
    u         (inflevel) int64 8B 3
    v         (inflevel) int64 8B 2

xarray.Dataset

• Dimensions:
  • inflevel: 1
• Coordinates: (2)
  • dim
    ()
    <U9
    'longitude'

    description :

    dimension of the source dataset along which the bitwise information has been analysed.

    array('longitude', dtype='<U9')

  • inflevel
    (inflevel)
    float64
    0.99

    array([0.99])

• Data variables: (3)
  • z
    (inflevel)
    int64
    10

    array([10])

  • u
    (inflevel)
    int64
    3

    array([3])

  • v
    (inflevel)
    int64
    2

    array([2])

• Indexes: (1)
  • inflevel
    PandasIndex

    PandasIndex(Index([0.99], dtype='float64', name='inflevel'))

• Attributes: (0)

Apply bitrounding

using xbitinfo.bitround.xr_bitround() or xbitinfo.bitround.jl_bitround() (does not work for chunked data)

ds_bitrounded = xb.xr_bitround(ds, keepbits)

xr.concat([ds, ds_bitrounded], "bitround").isel(level=0)["v"].plot(
    col="bitround", row="month"
)

<xarray.plot.facetgrid.FacetGrid at 0x7f3e35996000>

Save compressed

using xbitinfo.save_compressed.ToCompressed_Netcdf or xbitinfo.save_compressed.ToCompressed_Zarr

NetCDF

ds_bitrounded.to_compressed_netcdf("bitrounded_compressed.nc")

ds.to_compressed_netcdf("compressed.nc")

ds.to_netcdf("original.nc")

!du -hs *.nc

7.5M	0.air_original.nc
532K	bitrounded_compressed.nc
4.1M	compressed.nc
8.0M	original.nc

!rm *.nc

Zarr

ds_bitrounded.to_compressed_zarr("bitrounded_compressed.zarr", mode="w")
ds.to_compressed_zarr("compressed.zarr", mode="w")
ds.to_zarr(
    "original.zarr", mode="w", encoding={v: {"compressor": None} for v in ds.data_vars}
);

!du -hs *.zarr

812K	air_bitrounded.zarr
1.1M	air_bitrounded_by_chunks.zarr
7.7M	air_compressed.zarr
912K	bitrounded_compressed.zarr
4.8M	compressed.zarr
11M	original.zarr

!rm -r *.zarr