[WIP] removing pycall dependencies

ODINN-SciML · Jan 17, 2025 · dcfb65a · dcfb65a
1 parent 80444df
commit dcfb65a
Show file tree

Hide file tree

Showing 19 changed files with 150 additions and 230 deletions.
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -33,30 +33,30 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python }}  
-      - name: Create environment with micromamba 🐍🖤
-        uses: mamba-org/setup-micromamba@v1
-        with: 
-          micromamba-version: '1.5.6-0'
-          environment-file: ./environment.yml
-          environment-name: oggm_env                # it is recommendable to add both name and yml file. 
-          init-shell: bash
-          cache-environment: true
-#           condarc-file: ./condarc.yml                # If necessary, we can include .condarc to configure environment
-      - name: Test creation of environment with micromamba 🔧🐍🖤
-        run: | 
-            which python
-            conda env export
-        shell: bash -el {0}
-      - name: Update certifi 🔒🔑
-        run: | 
-            pip install --upgrade certifi
-        shell: bash -el {0}
-#       - name: Test OGGM installation 🔧🌎 
-#         run: pytest.oggm
+#       - name: Create environment with micromamba 🐍🖤
+#         uses: mamba-org/setup-micromamba@v1
+#         with: 
+#           micromamba-version: '1.5.6-0'
+#           environment-file: ./environment.yml
+#           environment-name: oggm_env                # it is recommendable to add both name and yml file. 
+#           init-shell: bash
+#           cache-environment: true
+# #           condarc-file: ./condarc.yml                # If necessary, we can include .condarc to configure environment
+#       - name: Test creation of environment with micromamba 🔧🐍🖤
+#         run: | 
+#             which python
+#             conda env export
+#         shell: bash -el {0}
+#       - name: Update certifi 🔒🔑
+#         run: | 
+#             pip install --upgrade certifi
+#         shell: bash -el {0}
+# #       - name: Test OGGM installation 🔧🌎 
+# #         run: pytest.oggm
+# #         shell: bash -el {0}
+#       - name: Set ENV Variables for PyCall.jl 🐍 📞
+#         run: export PYTHON=/home/runner/micromamba/envs/oggm_env/bin/python
 #         shell: bash -el {0}
-      - name: Set ENV Variables for PyCall.jl 🐍 📞
-        run: export PYTHON=/home/runner/micromamba/envs/oggm_env/bin/python
-        shell: bash -el {0}
       - uses: julia-actions/setup-julia@v1
         with:
           version: ${{ matrix.version }}

diff --git a/Project.toml b/Project.toml
@@ -32,7 +32,6 @@ PlotThemes = "ccf2f8ad-2431-5c83-bf29-c5338b663b6a"
 Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
 Polynomials = "f27b6e38-b328-58d1-80ce-0feddd5e7a45"
 ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca"
-PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0"
 PyPlot = "d330b81b-6aea-500a-939a-2ce795aea3ee"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
@@ -64,7 +63,6 @@ PlotThemes = "3.1"
 Plots = "1.2"
 Polynomials = "2, 3, 4"
 ProgressMeter = "1"
-PyCall = "1.9"
 PyPlot = "2.11"
 Revise = "3.5"
 SciMLSensitivity = "7.20"

diff --git a/README.md b/README.md
@@ -67,16 +67,19 @@ ODINN's architecture makes it really straightforward to retrieve all the necessa
 ```julia
 using ODINN
 
+# Data are automatically downloaded, retrieve the local paths
+rgi_paths = get_rgi_paths()
+
 # We create the necessary parameters
-params = Parameters(OGGM = OGGMparameters(working_dir=joinpath(homedir(), "OGGM/ODINN_tests")),
-		    simulation = SimulationParameters(working_dir=working_dir,
-						      tspan=(2010.0, 2015.0),
-						      workers=5),
+params = Parameters(simulation = SimulationParameters(working_dir=working_dir,
+									tspan=(2010.0, 2015.0),
+									workers=5,
+									rgi_paths=rgi_paths),
 		    hyper = Hyperparameters(batch_size=4,
 					    epochs=10,
 					    optimizer=ODINN.ADAM(0.01)),
 		    UDE = UDEparameters(target = "A")
-		   ) 
+		   )
 
 # We define which glacier RGI IDs we want to work with
 rgi_ids = ["RGI60-11.03638", "RGI60-11.01450", "RGI60-08.00213", "RGI60-04.04351"]
@@ -86,7 +89,7 @@ model = Model(iceflow = SIA2Dmodel(params),
 	      mass_balance = mass_balance = TImodel1(params; DDF=6.0/1000.0, acc_factor=1.2/1000.0),
 	      machine_learning = NN(params))
 
-# We initialize the glaciers with all the necessary data 
+# We initialize the glaciers with all the necessary data
 glaciers = initialize_glaciers(rgi_ids, params)
 
 # We specify the type of simulation we want to perform

diff --git a/scripts/MB_benchmark.jl b/scripts/MB_benchmark.jl
@@ -61,23 +61,17 @@ function run_benchmark(analysis)
 
     tspan = (2010.0,2015.0) # period in years for simulation
 
-    # Configure OGGM settings in all workers
-    # Use a separate working dir to avoid conflicts with other simulations
-    working_dir = joinpath(homedir(), "Python/OGGM_memory_benchmark")
-
     #########################################
     ###########  CLIMATE DATA  ##############
     #########################################
 
     if analysis == "timeit"
 
-        @timeit to "oggm config" oggm_config(working_dir; processes=1)
-
         # Defining glaciers to be modelled with RGI IDs
         # RGI60-11.03638 # Argentière glacier
         # RGI60-11.01450 # Aletsch glacier
         rgi_ids = ["RGI60-11.03638", "RGI60-11.01450"]
-    
+
         ### Initialize glacier directory to obtain DEM and ice thickness inversion  ###
         gdirs = @timeit to "init_gdirs" init_gdirs(rgi_ids)
 
@@ -109,13 +103,11 @@ function run_benchmark(analysis)
 
     elseif analysis == "profile"
 
-        @profview_allocs oggm_config(working_dir)
-
         # Defining glaciers to be modelled with RGI IDs
         # RGI60-11.03638 # Argentière glacier
         # RGI60-11.01450 # Aletsch glacier
         rgi_ids = ["RGI60-11.03638", "RGI60-11.01450"]
-    
+
         ### Initialize glacier directory to obtain DEM and ice thickness inversion  ###
         gdirs = init_gdirs(rgi_ids)
 
@@ -137,8 +129,6 @@ function run_benchmark(analysis)
 
     elseif analysis == "types"
 
-        oggm_config(working_dir)
-
         # Defining glaciers to be modelled with RGI IDs
         # RGI60-11.03638 # Argentière glacier
         # RGI60-11.01450 # Aletsch glacier

diff --git a/scripts/benchmark_ODINN.jl b/scripts/benchmark_ODINN.jl
@@ -96,11 +96,6 @@ function run_benchmark()
 
     tspan = (2010.0,2015.0) # period in years for simulation
 
-    # Configure OGGM settings in all workers
-    # Use a separate working dir to avoid conflicts with other simulations
-    working_dir = joinpath(homedir(), "Python/OGGM_data_benchmark")
-    oggm_config(working_dir)
-
     # Defining glaciers to be modelled with RGI IDs
     # RGI60-11.03638 # Argentière glacier
     # RGI60-11.01450 # Aletsch glacier

diff --git a/scripts/diffusivity_inversion_experiment.jl b/scripts/diffusivity_inversion_experiment.jl
@@ -42,14 +42,10 @@ function run()
 
     tspan = (2017, 2018) # period in years for simulation (also for spin-up)
 
-    # Configure OGGM settings in all workers
-    working_dir = joinpath(homedir(), "Python/OGGM_data_diffusivity")
-    oggm_config(working_dir)    
-
     # Defining glaciers to be modelled with RGI IDs
     rgi_ids = ["RGI60-11.03638", "RGI60-11.01450", "RGI60-08.00213", "RGI60-04.04351", "RGI60-01.02170",
-                "RGI60-02.05098", "RGI60-01.01104", "RGI60-01.09162", "RGI60-01.00570", "RGI60-04.07051",                	
-                "RGI60-07.00274", "RGI60-07.01323", "RGI60-03.04207", "RGI60-03.03533", "RGI60-01.17316", 
+                "RGI60-02.05098", "RGI60-01.01104", "RGI60-01.09162", "RGI60-01.00570", "RGI60-04.07051",
+                "RGI60-07.00274", "RGI60-07.01323", "RGI60-03.04207", "RGI60-03.03533", "RGI60-01.17316",
                 "RGI60-07.01193", "RGI60-01.22174", "RGI60-14.07309", "RGI60-15.10261"]
 
     ### Initialize glacier directory to obtain DEM and ice thickness inversion  ###

diff --git a/scripts/new_API_tests.jl b/scripts/new_API_tests.jl
@@ -12,6 +12,8 @@ nglaciers = 1
 
 function API_test(tspan)
 
+    rgi_paths = get_rgi_paths()
+
     to = get_timer("ODINN")
     reset_timer!(to)
 
@@ -23,10 +25,11 @@ function API_test(tspan)
                                                             use_MB=true,
                                                             use_iceflow=true,
                                                             multiprocessing=true,
-                                                            workers=1),
+                                                            workers=1,
+                                                            rgi_paths=rgi_paths),
                             physical=PhysicalParameters(A=2e-17),
                             solver=SolverParameters(reltol=1e-7))
-    # Create an ODINN model based on a 2D Shallow Ice Approximation, 
+    # Create an ODINN model based on a 2D Shallow Ice Approximation,
     # a TI model with 1 DDF, and a neural network
     model = Model(iceflow = SIA2Dmodel(parameters),
                   mass_balance = TImodel1(parameters; DDF=8.0/1000.0, acc_factor=1.0/1000.0),
@@ -40,7 +43,7 @@ function API_test(tspan)
     prediction = Prediction(model, glaciers, parameters)
 
     # We run the simulation
-    @timeit get_timer("ODINN") "Run prediction" run!(prediction) 
+    @timeit get_timer("ODINN") "Run prediction" run!(prediction)
     @show to
 
     display(ODINN.Makie.heatmap(prediction.results[1].S))

diff --git a/scripts/rheology_inversions.jl b/scripts/rheology_inversions.jl
@@ -25,7 +25,7 @@ processes = 10
 ODINN.enable_multiprocessing(processes)
 # Flags
 ODINN.set_use_MB(false)
-ODINN.make_plots(true)    
+ODINN.make_plots(true)
 # UDE training
 ODINN.set_train(true)    # Train UDE
 ODINN.set_retrain(false) # Re-use previous NN weights to continue training
@@ -34,10 +34,6 @@ function run()
 
     tspan = (2017, 2018) # period in years for simulation
 
-    # Configure OGGM settings in all workers
-    working_dir = joinpath(homedir(), "Python/OGGM_data_D")
-    oggm_config(working_dir)    
-
     gtd_file, rgi_ids = ODINN.get_glathida_path_and_IDs()
     @infiltrate
 

diff --git a/scripts/toy_model.jl b/scripts/toy_model.jl
@@ -18,7 +18,7 @@ using Plots
 using Infiltrator
 using Distributed
 using JLD2
-using Statistics  
+using Statistics
 # using AbbreviatedStackTraces
 
 # Activate to avoid GKS backend Plot issues in the JupyterHub
@@ -32,8 +32,8 @@ function run_toy_model()
     ODINN.enable_multiprocessing(processes)
     # Flags
     ODINN.set_use_MB(true)
-    ODINN.make_plots(true)    
-    # Spin up 
+    ODINN.make_plots(true)
+    # Spin up
     ODINN.set_run_spinup(false) # Run the spin-up random_MB = generate_random_MB(gdirs_climate, tspan; plot=false)n
     ODINN.set_use_spinup(false) # Use the updated spinup
     # Reference simulations
@@ -47,14 +47,10 @@ function run_toy_model()
 
     tspan = (2010.0, 2015.0) # period in years for simulation
 
-    # Configure OGGM settings in all workers
-    working_dir = joinpath(homedir(), "Python/OGGM_data")
-    oggm_config(working_dir)
-
     # Defining glaciers to be modelled with RGI IDs
     rgi_ids = ["RGI60-11.03638", "RGI60-11.01450", "RGI60-08.00213", "RGI60-04.04351", "RGI60-01.02170",
-                "RGI60-02.05098", "RGI60-01.01104", "RGI60-01.09162", "RGI60-01.00570", "RGI60-04.07051",                	
-                "RGI60-07.00274", "RGI60-07.01323", "RGI60-03.04207", "RGI60-03.03533", "RGI60-01.17316"]#, 
+                "RGI60-02.05098", "RGI60-01.01104", "RGI60-01.09162", "RGI60-01.00570", "RGI60-04.07051",
+                "RGI60-07.00274", "RGI60-07.01323", "RGI60-03.04207", "RGI60-03.03533", "RGI60-01.17316"]#,
                 # "RGI60-07.01193", "RGI60-01.22174", "RGI60-14.07309", "RGI60-15.10261"]
 
     # rgi_ids = rgi_ids[1:2]

diff --git a/src/ODINN.jl b/src/ODINN.jl
@@ -26,7 +26,6 @@ Plots.theme(:wong2) # sets overall theme for Plots
 import Pkg
 using Distributed
 using ProgressMeter
-using PyCall
 using Downloads
 using TimerOutputs
 using GeoStats
@@ -41,28 +40,6 @@ const global root_dir::String = dirname(Base.current_project())
 const global root_plots::String = joinpath(root_dir, "plots")
 
 
-# ##############################################
-# ############  PYTHON LIBRARIES  ##############
-# ##############################################
-
-# We either retrieve the reexported Python libraries from Sleipnir or we start from scratch
-const netCDF4::PyObject = isdefined(Sleipnir, :netCDF4) ? Sleipnir.netCDF4 : PyNULL()
-const cfg::PyObject = isdefined(Sleipnir, :cfg) ? Sleipnir.cfg : PyNULL()
-const utils::PyObject = isdefined(Sleipnir, :utils) ? Sleipnir.utils : PyNULL()
-const workflow::PyObject = isdefined(Sleipnir, :workflow) ? Sleipnir.workflow : PyNULL()
-const tasks::PyObject = isdefined(Sleipnir, :tasks) ? Sleipnir.tasks : PyNULL()
-const global_tasks::PyObject = isdefined(Sleipnir, :global_tasks) ? Sleipnir.global_tasks : PyNULL()
-const graphics::PyObject = isdefined(Sleipnir, :graphics) ? Sleipnir.graphics : PyNULL()
-const bedtopo::PyObject = isdefined(Sleipnir, :bedtopo) ? Sleipnir.bedtopo : PyNULL()
-const millan22::PyObject = isdefined(Sleipnir, :millan22) ? Sleipnir.millan22 : PyNULL()
-const MBsandbox::PyObject = isdefined(Sleipnir, :MBsandbox) ? Sleipnir.MBsandbox : PyNULL()
-const salem::PyObject = isdefined(Sleipnir, :salem) ? Sleipnir.salem : PyNULL()
-
-# Essential Python libraries
-const xr::PyObject = isdefined(Sleipnir, :xr) ? Sleipnir.xr : PyNULL()
-const rioxarray::PyObject = isdefined(Sleipnir, :rioxarray) ? Sleipnir.rioxarray : PyNULL()
-const pd::PyObject = isdefined(Sleipnir, :pd) ? Sleipnir.pd : PyNULL()
-
 # ##############################################
 # ############  ODINN LIBRARIES  ###############
 # ##############################################