From a537f248e4ac2d177300b5a48ce8dcee4ead3359 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi?= <remi.cresson@inrae.fr> Date: Wed, 24 Apr 2024 16:09:55 +0200 Subject: [PATCH 1/8] ci: enh --- .gitlab-ci.yml | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bd41bbf9..002a22c1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -94,14 +94,21 @@ docker image: stage: Static Analysis allow_failure: true -flake8: + +.py_static_analysis_base: extends: .static_analysis_base + rules: + - changes: + - otbtf/**/* + +flake8: + extends: .py_static_analysis_base script: - sudo pip install flake8 - flake8 $OTBTF_SRC/otbtf --exclude=tensorflow_v1x pylint: - extends: .static_analysis_base + extends: .py_static_analysis_base script: - sudo pip install pylint - pylint $OTBTF_SRC/otbtf --ignore=tensorflow_v1x @@ -115,6 +122,10 @@ codespell: cppcheck: extends: .static_analysis_base + rules: + - changes: + - app/**/* + - include/**/* script: - sudo apt update && sudo apt install cppcheck -y - cd $OTBTF_SRC/ && cppcheck --enable=all --error-exitcode=1 -I include/ --suppress=missingInclude --suppress=unusedFunction . @@ -123,6 +134,13 @@ cppcheck: stage: Documentation before_script: - pip install -r doc/doc_requirements.txt + rules: + - changes: + - doc/**/* + - *.md + - *.txt + - mkdocs.yml + - .readthedocs.yaml artifacts: paths: - public @@ -147,6 +165,18 @@ pages: .tests_base: tags: [godzilla] + rules: + - changes: + - app/**/* + - include/**/* + - otbtf/**/* + - test/**/* + - tools/**/* + - .gitlab-ci.yml + - Dockerfile + - otb-module.cmake + - CMakeLists.txt + - setup.py artifacts: paths: - $ARTIFACT_TEST_DIR/*.* @@ -260,7 +290,7 @@ deploy_cpu-dev-testing: extends: .docker_build_base stage: Ship only: - - master + - tags deploy_cpu: extends: .ship base -- GitLab From 8ca0392657a2455f9033b968c9c99765f5a8fe82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi?= <remi.cresson@inrae.fr> Date: Wed, 24 Apr 2024 16:10:04 +0200 Subject: [PATCH 2/8] doc: update links --- doc/app_sampling.md | 2 +- doc/app_training.md | 2 +- doc/docker_use.md | 22 ++++++++++++---------- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/doc/app_sampling.md b/doc/app_sampling.md index b79ba7f4..9a9fc047 100644 --- a/doc/app_sampling.md +++ b/doc/app_sampling.md @@ -84,7 +84,7 @@ specific field of the input vector data. Typically, the *class* field can be used to generate a dataset suitable for a model that performs pixel wise classification. - + The application description can be displayed using: diff --git a/doc/app_training.md b/doc/app_training.md index eb3f4708..c1e731ef 100644 --- a/doc/app_training.md +++ b/doc/app_training.md @@ -47,7 +47,7 @@ patches images, a convenient method consist in reading patches images as numpy arrays using OTB applications (e.g. `ExtractROI`) or GDAL, then do a `numpy.reshape` to the dimensions wanted. - + The application description can be displayed using: diff --git a/doc/docker_use.md b/doc/docker_use.md index 81211066..323e4268 100644 --- a/doc/docker_use.md +++ b/doc/docker_use.md @@ -21,16 +21,18 @@ Read more in the following sections. Here is the list of the latest OTBTF docker images hosted on [dockerhub](https://hub.docker.com/u/mdl4eo). Since OTBTF >= 3.2.1 you can find the latest docker images on -[gitlab.irstea.fr](https://gitlab.irstea.fr/remi.cresson/otbtf/container_registry). - -| Name | Os | TF | OTB | Description | Dev files | Compute capability | -|------------------------------------------------------------------------------------| ------------- |-------|-------| ---------------------- | --------- | ------------------ | -| **mdl4eo/otbtf:4.3.1-cpu** | Ubuntu Jammy | r2.14 | 9.0.0 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6| -| **mdl4eo/otbtf:4.3.1-cpu-dev** | Ubuntu Jammy | r2.14 | 9.0.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6| -| **mdl4eo/otbtf:4.3.1-gpu** | Ubuntu Jammy | r2.14 | 9.0.0 | GPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6| -| **mdl4eo/otbtf:4.3.1-gpu-dev** | Ubuntu Jammy | r2.14 | 9.0.0 | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6| -| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:4.3.1-gpu-opt** | Ubuntu Jammy | r2.14 | 9.0.0 | GPU with opt. | no | 5.2,6.1,7.0,7.5,8.6| -| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:4.3.1-gpu-opt-dev** | Ubuntu Jammy | r2.14 | 9.0.0 | GPU with opt. (dev) | yes | 5.2,6.1,7.0,7.5,8.6| +[gitlab.irstea.fr](https://gitlab.irstea.fr/remi.cresson/otbtf/container_registry) +(before otbtf 4.3.0) and [forgemia](https://forgemia.inra.fr/orfeo-toolbox/otbtf/container_registry) +(since otbtf 4.3.1). + +| Name | Os | TF | OTB | Description | Dev files | Compute capability | +|---------------------------------------------------------------------| ------------- |-------|-------| ---------------------- | --------- | ------------------ | +| **mdl4eo/otbtf:4.3.1-cpu** | Ubuntu Jammy | r2.14 | 9.0.0 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6| +| **mdl4eo/otbtf:4.3.1-cpu-dev** | Ubuntu Jammy | r2.14 | 9.0.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6| +| **mdl4eo/otbtf:4.3.1-gpu** | Ubuntu Jammy | r2.14 | 9.0.0 | GPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6| +| **mdl4eo/otbtf:4.3.1-gpu-dev** | Ubuntu Jammy | r2.14 | 9.0.0 | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6| +| **registry.forgemia.inra.fr/orfeo-toolbox/otbtf:4.3.1-gpu-opt** | Ubuntu Jammy | r2.14 | 9.0.0 | GPU with opt. | no | 5.2,6.1,7.0,7.5,8.6| +| **registry.forgemia.inra.fr/orfeo-toolbox/otbtf:4.3.1-gpu-opt-dev** | Ubuntu Jammy | r2.14 | 9.0.0 | GPU with opt. (dev) | yes | 5.2,6.1,7.0,7.5,8.6| The list of older releases is available [here](#older-images). -- GitLab From 45ceeb8f79589077f9b4b37778cd02e8492abe0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi?= <remi.cresson@inrae.fr> Date: Wed, 24 Apr 2024 16:10:18 +0200 Subject: [PATCH 3/8] doc: update links --- otbtf/examples/tensorflow_v1x/__init__.py | 10 +++++----- setup.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/otbtf/examples/tensorflow_v1x/__init__.py b/otbtf/examples/tensorflow_v1x/__init__.py index c77256a4..796e4cbd 100644 --- a/otbtf/examples/tensorflow_v1x/__init__.py +++ b/otbtf/examples/tensorflow_v1x/__init__.py @@ -38,7 +38,7 @@ Predicted label is a single pixel, for an input patch of size 16x16 (for an inpu The learning rate of the training operator can be adjusted using the *lr* placeholder. The following figure summarizes this architecture. -<img src ="https://gitlab.irstea.fr/remi.cresson/otbtf/-/raw/develop/doc/images/savedmodel_simple_cnn.png" /> +<img src ="https://forgemia.inra.fr/orfeo-toolbox/otbtf/-/raw/develop/doc/images/savedmodel_simple_cnn.png" /> ## Generate the model @@ -143,7 +143,7 @@ otbcli_TensorflowModelServe \\ The `create_savedmodel_simple_fcn.py` script enables you to create a fully convolutional model which does not use any stride. -<img src ="https://gitlab.irstea.fr/remi.cresson/otbtf/-/raw/develop/doc/images/savedmodel_simple_fcnn.png" /> +<img src ="https://forgemia.inra.fr/orfeo-toolbox/otbtf/-/raw/develop/doc/images/savedmodel_simple_fcnn.png" /> Thanks to that, once trained this model can be applied on the image to produce a landcover map at the same resolution as the input image, in a fully @@ -208,7 +208,7 @@ available parameters. Let's train the M3 model from time series (TS) and Very High Resolution Satellite (VHRS) patches images. -<img src ="https://gitlab.irstea.fr/remi.cresson/otbtf/-/raw/develop/doc/images/model_training.png" /> +<img src ="https://forgemia.inra.fr/orfeo-toolbox/otbtf/-/raw/develop/doc/images/model_training.png" /> First, tell OTBTF that we want two sources: one for time series + one for VHR image @@ -255,7 +255,7 @@ otbcli_TensorflowModelTrain \\ Let's produce a land cover map using the M3 model from time series (TS) and Very High Resolution Satellite image (VHRS) -<img src ="https://gitlab.irstea.fr/remi.cresson/otbtf/-/raw/develop/doc/images/classif_map.png" /> +<img src ="https://forgemia.inra.fr/orfeo-toolbox/otbtf/-/raw/develop/doc/images/classif_map.png" /> Since we provide time series as the reference source (*source1*), the output classes are estimated at the same resolution. This model can be run in @@ -363,7 +363,7 @@ See: Gaetano, R., Ienco, D., Ose, K., & Cresson, R. (2018). *A two-branch CNN architecture for land cover classification of PAN and MS imagery*. Remote Sensing, 10(11), 1746. -<img src ="https://gitlab.irstea.fr/remi.cresson/otbtf/-/raw/develop/doc/images/savedmodel_simple_pxs_fcn.png" /> +<img src ="https://forgemia.inra.fr/orfeo-toolbox/otbtf/-/raw/develop/doc/images/savedmodel_simple_pxs_fcn.png" /> Use `create_savedmodel_pxs_fcn.py` to generate this model. diff --git a/setup.py b/setup.py index 8519fb11..c1a6294a 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ setuptools.setup( description="OTBTF: Orfeo ToolBox meets TensorFlow", long_description=long_description, long_description_content_type="text/markdown", - url="https://gitlab.irstea.fr/remi.cresson/otbtf", + url="https://github.com/remicres/otbtf", classifiers=[ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", -- GitLab From 8869511fbca39f0096d9a65cc7622fac00a25238 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi?= <remi.cresson@inrae.fr> Date: Wed, 24 Apr 2024 16:12:41 +0200 Subject: [PATCH 4/8] ci: enh --- .gitlab-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 002a22c1..64f888d4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -137,8 +137,7 @@ cppcheck: rules: - changes: - doc/**/* - - *.md - - *.txt + - "*.{md,txt}" - mkdocs.yml - .readthedocs.yaml artifacts: -- GitLab From 01ca6840d333e5704ded6d599f617895f63b2568 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi?= <remi.cresson@inrae.fr> Date: Wed, 24 Apr 2024 16:15:16 +0200 Subject: [PATCH 5/8] ci: enh --- .gitlab-ci.yml | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 64f888d4..a0c9c1e1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,7 +24,7 @@ image: $BRANCH_IMAGE workflow: rules: - if: $CI_MERGE_REQUEST_ID || $CI_COMMIT_REF_NAME =~ /master/ # Execute jobs in merge request context, or commit in master branch - + stages: - Build - Static Analysis @@ -36,7 +36,7 @@ stages: .docker_build_base: allow_failure: false - tags: [godzilla] + tags: [ godzilla ] image: docker:latest services: - name: docker:dind @@ -99,7 +99,7 @@ docker image: extends: .static_analysis_base rules: - changes: - - otbtf/**/* + - otbtf/**/* flake8: extends: .py_static_analysis_base @@ -124,8 +124,8 @@ cppcheck: extends: .static_analysis_base rules: - changes: - - app/**/* - - include/**/* + - app/**/* + - include/**/* script: - sudo apt update && sudo apt install cppcheck -y - cd $OTBTF_SRC/ && cppcheck --enable=all --error-exitcode=1 -I include/ --suppress=missingInclude --suppress=unusedFunction . @@ -136,10 +136,10 @@ cppcheck: - pip install -r doc/doc_requirements.txt rules: - changes: - - doc/**/* - - "*.{md,txt}" - - mkdocs.yml - - .readthedocs.yaml + - doc/**/* + - "*.{md,txt}" + - mkdocs.yml + - .readthedocs.yaml artifacts: paths: - public @@ -163,19 +163,19 @@ pages: - public .tests_base: - tags: [godzilla] + tags: [ godzilla ] rules: - changes: - - app/**/* - - include/**/* - - otbtf/**/* - - test/**/* - - tools/**/* - - .gitlab-ci.yml - - Dockerfile - - otb-module.cmake - - CMakeLists.txt - - setup.py + - app/**/* + - include/**/* + - otbtf/**/* + - test/**/* + - tools/**/* + - .gitlab-ci.yml + - Dockerfile + - otb-module.cmake + - CMakeLists.txt + - setup.py artifacts: paths: - $ARTIFACT_TEST_DIR/*.* @@ -186,7 +186,7 @@ ctest: extends: .tests_base stage: Test script: - - sudo apt update && sudo apt install -y git-lfs + - sudo apt update && sudo apt install -y git-lfs - cd /src/otb/otb && sudo git lfs fetch --all && sudo git lfs pull - cd $OTB_BUILD/ - sudo ctest -L OTBTensorflow @@ -212,7 +212,7 @@ crc_book: - TMPDIR=$CRC_BOOK_TMP python -m pytest --junitxml=$CI_PROJECT_DIR/report_tutorial.xml $OTBTF_SRC/test/tutorial_unittest.py after_script: - cp $CRC_BOOK_TMP/*.* $ARTIFACT_TEST_DIR/ - + sr4rs: extends: .applications_test_base script: -- GitLab From 579bafb7bdc26f655eaadc9042d1761c1a885a3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi?= <remi.cresson@inrae.fr> Date: Wed, 24 Apr 2024 16:16:27 +0200 Subject: [PATCH 6/8] ci: enh --- .gitlab-ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a0c9c1e1..7a8f130e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -137,9 +137,7 @@ cppcheck: rules: - changes: - doc/**/* - - "*.{md,txt}" - - mkdocs.yml - - .readthedocs.yaml + artifacts: paths: - public -- GitLab From cb9b92822de4a508e3022784db51d4e1dfc44021 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi?= <remi.cresson@inrae.fr> Date: Wed, 24 Apr 2024 16:26:29 +0200 Subject: [PATCH 7/8] ci: enh --- .gitlab-ci.yml | 35 +++++++++++++---------------------- 1 file changed, 13 insertions(+), 22 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7a8f130e..2becb355 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -130,35 +130,26 @@ cppcheck: - sudo apt update && sudo apt install cppcheck -y - cd $OTBTF_SRC/ && cppcheck --enable=all --error-exitcode=1 -I include/ --suppress=missingInclude --suppress=unusedFunction . -.doc_base: +pages: stage: Documentation - before_script: - - pip install -r doc/doc_requirements.txt + variables: + PTH: "public_test" rules: - changes: - doc/**/* - - artifacts: - paths: - - public - - public_test - -pages_test: - extends: .doc_base - except: - - master - script: - - mkdocs build --site-dir public_test - -pages: - extends: .doc_base - only: - - master + - "*.{md,txt}" + - mkdocs.yml + - .readthedocs.yaml + - if: $CI_COMMIT_REF_NAME == /master/ + variables: + PTH: "public" + before_script: + - pip install -r doc/doc_requirements.txt script: - - mkdocs build --site-dir public + - mkdocs build --site-dir $PTH artifacts: paths: - - public + - $PTH .tests_base: tags: [ godzilla ] -- GitLab From 3d58d28a458da057f3297a59d3b9fcfbcf7a237a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi?= <remi.cresson@inrae.fr> Date: Wed, 24 Apr 2024 16:30:11 +0200 Subject: [PATCH 8/8] ci: enh --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2becb355..17526bf7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -45,7 +45,7 @@ stages: timeout: 10 hours -docker image: +dev docker image build: extends: .docker_build_base stage: Build except: -- GitLab