Ticket #2002: 0001-ticket-2002-support-subdataset-levels-crs-filtering-.patch

applications/wcst_import/recipes_custom/sentinel2/recipe.py

@@ -26 +26 @@
 from master.importer.importer import Importer
 from master.importer.multi_importer import MultiImporter
 from master.error.runtime_exception import RuntimeException
+from master.error.validate_exception import RecipeValidationException
 from master.evaluator.sentence_evaluator import SentenceEvaluator
 from master.evaluator.expression_evaluator_factory import ExpressionEvaluatorFactory
 from master.helper.user_band import UserBand
@@ -48 +49 @@
     # supported product levels
     LVL_L1C = 'L1C'
     LVL_L2A = 'L2A'
+    LEVELS = [LVL_L1C, LVL_L2A]
+
     # resolutions in a single Sentinel 2 dataset; TCI (True Color Image) is 10m
     RES_10m = '10m'
     RES_20m = '20m'
     RES_60m = '60m'
     RES_TCI = 'TCI'
-
-    RES_DICT = {RES_10m: [1, 10, -10], RES_20m: [1, 20, -20], RES_60m: [1, 60, -60], RES_TCI: [1, 10, -10]}
+    # resolution (subdataset name) -> actual resolution numbers
+    RES_DICT = {RES_10m: [1, 10, -10], 
+                RES_20m: [1, 20, -20],
+                RES_60m: [1, 60, -60],
+                RES_TCI: [1, 10, -10]}
+    # list of subdatasets to import
+    SUBDATASETS = [RES_10m, RES_20m, RES_60m, RES_TCI]
 
     # variables that can be used to template the coverage id
     VAR_CRS_CODE = '${crsCode}'
@@ -98 +106 @@
         RES_TCI: BANDS_L1C[RES_TCI],
     }
     BANDS = { LVL_L1C: BANDS_L1C, LVL_L2A: BANDS_L2A }
-    # number of subdatasets in a Sentinel 2 dataset
-    SUBDATASETS = 4
     DEFAULT_CRS = "OGC/0/AnsiDate@EPSG/0/${crsCode}"
     DEFAULT_IMPORT_ORDER = GdalToCoverageConverter.IMPORT_ORDER_ASCENDING
     
@@ -110 +116 @@
     def __init__(self, session):
         super(Recipe, self).__init__(session)
         self._init_options()
-        # subdatasets have a specific path scheme and prepending "file://" interferes with it
-        # TODO: however uncommenting the below causes another error:
-        #       The URL provided in the coverageRef parameter is malformed.
-        # ConfigManager.root_url = ""
 
     def validate(self):
         super(Recipe, self).validate()
+        if len(self.resolutions) == 0:
+            raise RecipeValidationException("No resolutions to import provided.")
+        for res in self.resolutions:
+            if res not in self.SUBDATASETS:
+                raise RecipeValidationException("Invalid resolution '" + str(res) + 
+                    "' provided, expected a subset of " + str(self.SUBDATASETS))
+        for lvl in self.levels:
+            if lvl not in self.LEVELS:
+                raise RecipeValidationException("Invalid level '" + str(lvl) + 
+                    "' provided, expected a subset of " + str(self.LEVELS))
 
     def describe(self):
         log.info("The recipe has been validated and is ready to run.")
@@ -152 +164 @@
     
     def _init_options(self):
         self._init_coverage_options()
+        self._init_input_options()
         self.coverage_id = self.session.get_coverage_id()
         self.import_order = self._set_option(self.options, 'import_order', self.DEFAULT_IMPORT_ORDER)
         self.wms_import = self._set_option(self.options, 'wms_import', False)
@@ -163 +176 @@
         self.crs = self._set_option(covopts, 'crs', self.DEFAULT_CRS)
         self._set_option(covopts, 'slicer', {})
         self._init_slicer_options(covopts)
+
+    def _init_input_options(self):
+        # specify a subset of resolutions to ingest
+        inputopts = self.session.get_input()
+        self.resolutions = self._set_option(inputopts, 'resolutions', None)
+        if self.resolutions is None:
+            self.resolutions = self._set_option(inputopts, 'subdatasets', None)
+        if self.resolutions is None:
+            self.resolutions = self.SUBDATASETS
+        # allow to ingest data with only particular crss
+        self.crss = self._set_option(inputopts, 'crss', [])
+        # ingest data if it's the specified levels
+        self.levels = self._set_option(inputopts, 'levels', [])
     
     def _init_slicer_options(self, covopts):
         sliceropts = covopts['slicer']
         self._set_option(sliceropts, 'type', 'gdal')
         self._set_option(sliceropts, 'pixelIsPoint', False)
-        if 'axes' not in sliceropts:
-            self._init_axes_options(sliceropts)
+        axesopts = self._init_axes_options()
+        if 'axes' in sliceropts:
+            for axis in sliceropts['axes']:
+                if axis not in axesopts:
+                    raise RecipeValidationException("Invalid axis '" + axis + "', expected one of ansi/E/N.")
+                for k in sliceropts['axes'][axis]:
+                    axesopts[axis][k] = sliceropts['axes'][axis][k]
+        sliceropts['axes'] = axesopts
     
-    def _init_axes_options(self, sliceropts):
-        sliceropts['axes'] = {
+    def _init_axes_options(self):
+        return {
             'ansi': {
                 "min": "datetime(regex_extract('${file:path}', '.*?/S2[^_]+_MSI[^_]+_([\\d]+)T[\\d]+_', 1), 'YYYYMMDD')",
                 "gridOrder": 0,
@@ -230 +262 @@
             gdal_ds.close()
 
             level = self._get_level(f.get_filepath())
+            if len(self.levels) > 0 and level not in self.levels:
+                # skip file, as it's not in the list of levels provided in the ingredients file
+                log.debug("Skipping " + level + " data")
+                continue
             crs_code = ""
 
             evaluator_slice = None
 
-            for res in [self.RES_10m, self.RES_20m, self.RES_60m, self.RES_TCI]:
+            for res in self.resolutions:
                 subds_file = self._get_subdataset_file(subdatasets, res)
                 crs_code = self._get_crs_code(subds_file.get_filepath(), crs_code)
+                if len(self.crss) > 0 and crs_code not in self.crss:
+                    # skip CRS, it's not in the list of CRSs provided in the ingredients file
+                    log.debug("Skipping data with CRS " + crs_code)
+                    continue 
                 cov_id = self._get_coverage_id(self.coverage_id, crs_code, level, res)
                 conv = self._get_convertor(convertors, cov_id, crs_code, level, res)
 
@@ -263 +303 @@
     
     def _get_subdatasets(self, gdal_ds, f):
         subdatasets = gdal_ds.get_subdatasets()
-        if len(subdatasets) != self.SUBDATASETS:
+        if len(subdatasets) != len(self.SUBDATASETS):
             raise RuntimeException("Cannot handle Sentinel 2 file " + f.get_filepath() + 
                                    ": GDAL reported " + str(len(subdatasets)) + 
-                                   " subdatasets, expected " + str(self.SUBDATASETS) + ".")
+                                   " subdatasets, expected " + str(len(self.SUBDATASETS)) + ".")
         return [name for (name, _) in subdatasets]
 
     def _get_subdataset_file(self, subdatasets, res):

applications/wcst_import/session.py

@@ -53 +53 @@
         self.files = self.parse_input(inp['paths'] if 'paths' in inp else [])
         self.coverage_id = inp['coverage_id'] if 'coverage_id' in inp else None
         self.recipe = recipe
+        self.input = inp
         self.wcs_service = config['service_url'] if "service_url" in config else None
         if "tmp_directory" in config:
             self.tmp_directory = config['tmp_directory']
@@ -258 +259 @@
         """
         return self.recipe
 
+    def get_input(self):
+        """
+        Returns the input section of the ingredients
+        :rtype dict[str,str]
+        """
+        return self.input
+
     def get_coverage_id(self):
         """
         Returns the coverage id for this session

doc/main/05_geo-services-guide.rst

@@ -1604 +1604 @@
 * :ref:`Irregular timeseries <data-import-recipe-irregular-timeseries>`
 * :ref:`General coverage <data-import-recipe-general-coverage>`
 * :ref:`Import from external WCS <data-import-recipe-wcs_extract>`
+* Specialized recipes
+
+    - :ref:`Sentinel 2 <data-import-recipe-sentinel2>`
 
 For each one of these there is an ingredients example under the
 `ingredients/ <http://rasdaman.org/browser/applications/wcst_import/ingredients>`_
@@ -2560 +2563 @@
     {
       "config": {
         "service_url": "http://localhost:8080/rasdaman/ows",
-        "tmp_directory": "/tmp/",
         "default_crs": "http://localhost:8080/def/crs/EPSG/0/4326",
-        "mock": false,
-        "automated": true,
-        "track_files": false
+        "automated": true
       },
       "input": {
         "coverage_id": "test_wcs_extract"
@@ -2588 +2588 @@
     }
 
 
+.. _data-import-recipe-sentinel2:
+
+Import Sentinel 2 data
+^^^^^^^^^^^^^^^^^^^^^^
+
+This is a convenience recipe for importing Sentinel 2 data in particular. It
+relies on support for Sentinel 2 in `more recent GDAL versions
+<https://gdal.org/frmt_sentinel2.html>`__. Importing zipped Sentinel 2 is also
+possible and automatically handled.
+
+Below is an example:
+
+.. code-block:: json
+
+    {
+      "config": {
+        "service_url": "http://localhost:8080/rasdaman/ows",
+        "automated": true
+      },
+      "input": {
+        "coverage_id": "S2_${crsCode}_${resolution}_${level}",
+        "paths": [ "S2*.zip" ],
+        // Optional filtering settings
+        "resolutions": ["10m", "20m", "60m", "TCI"],
+        "levels": ["L1C", "L2A"],
+        "crss": ["32757"] // remove or leave empty to ingest any CRS
+      },
+      "recipe": {
+        "name": "sentinel2",
+        "options": {
+          "coverage": {
+            "metadata": {
+              "type": "xml",
+              "global": {
+                "Title": "'Sentinel-2 data served by rasdaman'"
+              }
+            }
+          },
+          "tiling": "ALIGNED [0:0, 0:1999, 0:1999] TILE SIZE 32000000",
+          "wms_import": true
+        }
+      }
+    }
+
+The recipe extends `general_coverage <data-import-recipe-wcs_extract>`_ so
+the ``"recipe"`` section has the same structure. However, a lot of information
+is automatically filled in by the recipe now, so the ingredients file is much
+simpler as the example above shows.
+
+The other obvious difference is that the ``"coverage_id"`` is templated with
+several variables enclosed in ``${`` and ``}`` which are automatically replaced
+to generate the actual coverage name during import:
+
+- ``crsCode`` - the CRS EPSG code of the imported files, e.g. ``32757`` for 
+  WGS 84 / UTM zone 57S.
+
+- ``resolution`` - Sentinel 2 products bundle several subdatasets of different
+  resolutions: ``10m``, ``20m``, ``60m``, and ``TCI`` (True Color Image, also 
+  10m as it is derived from the B02, B03, and B04 10m bands).
+
+- ``level`` - ``L1C`` or ``L2A``
+
+If the files collected by ``"paths"`` are varying in any of these parameters,
+the corresponding variables must appear somewhere in the ``"coverage_id"``.
+Otherwise, the ingestion will either fail or result in invalid coverages. E.g.
+if all data is level ``L1C`` with CRS ``32757``, but still different
+resolutions, the ``"coverage_id"`` could be ``"MyCoverage_${resolution}"``;
+the other variables can still be specified though, so 
+``"MyCoverage_${resolution}_${crsCode}"`` is valid as well.
+
+In addition, the data to be ingested can be optionall filtered with the
+following options in the ``"input"`` section:
+
+- ``resolutions`` - specify a subset of resolutions to ingest from the data,
+  e.g. only the "10m" subdataset; if not specified, data of all supported
+  resolutions will be ingested.
+
+- ``levels`` - specify a subset of levels to ingest, so that files of other 
+  levels will be fully skipped; if not specified, data of all supported levels
+  will be ingested.
+
+- ``crss`` - specify a subset of CRSs to ingest; if not specified or empty
+  data of any CRS will be ingested.
+
+
 .. _wms-image-pyramids:
 
 Image pyramids