Skip to content
Snippets Groups Projects
Commit fcd9f807 authored by Max Melchert's avatar Max Melchert
Browse files

feat: global conda installation; local conda env

parent 189360e6
Branches
No related tags found
No related merge requests found
...@@ -5,8 +5,8 @@ SERVER_USER := mmustermann ...@@ -5,8 +5,8 @@ SERVER_USER := mmustermann
SERVER_HOST := hyrican-1-extern SERVER_HOST := hyrican-1-extern
## Python Environment ## Python Environment
CONDA_PATH := ~/.local/opt/conda CONDA_PATH := /opt/conda
CONDA_ENV := ml CONDA_ENV := ~/envs/ml
PYTHON_VERSION := 3.10 PYTHON_VERSION := 3.10
PYTHON_PACKAGES := tensorflow-cpu ipympl mypy jupyter pystan \ PYTHON_PACKAGES := tensorflow-cpu ipympl mypy jupyter pystan \
matplotlib seaborn scikit-learn openpyxl odfpy pyarrow \ matplotlib seaborn scikit-learn openpyxl odfpy pyarrow \
...@@ -28,13 +28,14 @@ REMOTE_PROJECT_DIR := ~/my_project ...@@ -28,13 +28,14 @@ REMOTE_PROJECT_DIR := ~/my_project
REMOTE_OUTPUT_FILES := output/* REMOTE_OUTPUT_FILES := output/*
## Misc ## Misc
COPY_LINKS_AS_FILES ?= false COPY_LINKS_AS_FILES := false
# --- # ---
LOGIN := $(SERVER_USER)@$(SERVER_HOST) LOGIN := $(SERVER_USER)@$(SERVER_HOST)
SSH := ssh $(LOGIN) SSH := ssh $(LOGIN)
RSYNC := rsync --archive --compress --progress $(if $(filter true,$(COPY_LINKS_AS_FILES)),--copy-links) RSYNC := rsync --archive --compress --progress $(if $(filter true,$(COPY_LINKS_AS_FILES)),--copy-links)
MAMBA := source $(CONDA_PATH)/etc/profile.d/conda.sh && mamba # TODO: mamba.sh
CONDA := source $(CONDA_PATH)/etc/profile.d/conda.sh && conda CONDA := source $(CONDA_PATH)/etc/profile.d/conda.sh && conda
.PHONY: install upload run kill log download delete .PHONY: install upload run kill log download delete
...@@ -44,7 +45,7 @@ help: # list targets and usage ...@@ -44,7 +45,7 @@ help: # list targets and usage
install: # install conda environment install: # install conda environment
$(SSH) '\ $(SSH) '\
$(CONDA) create -n $(CONDA_ENV) python=$(PYTHON_VERSION) $(PYTHON_PACKAGES)\ $(MAMBA) create -p $(CONDA_ENV) python=$(PYTHON_VERSION) $(PYTHON_PACKAGES)\
' '
upload: # upload project files from client to server upload: # upload project files from client to server
...@@ -53,7 +54,7 @@ upload: # upload project files from client to server ...@@ -53,7 +54,7 @@ upload: # upload project files from client to server
run: # execute main script on server run: # execute main script on server
$(SSH) '\ $(SSH) '\
cd $(REMOTE_PROJECT_DIR);\ cd $(REMOTE_PROJECT_DIR);\
$(CONDA) activate $(CONDA_ENV);\ $(CONDA) activate -p $(CONDA_ENV);\
python $(MAIN_SCRIPT);\ python $(MAIN_SCRIPT);\
$(CONDA) deactivate;\ $(CONDA) deactivate;\
' '
......
# Configuration
## Server
SERVER_USER := mmustermann
SERVER_HOST := hyrican-1-extern
## Python Environment
CONDA_PATH := ~/.local/opt/conda
CONDA_ENV := ml
PYTHON_VERSION := 3.10
PYTHON_PACKAGES := tensorflow-cpu ipympl mypy jupyter pystan \
matplotlib seaborn scikit-learn openpyxl odfpy pyarrow \
geopandas descartes pyogrio optuna plotly tqdm spyder \
jax-jumpy moviepy shimmy-atari gymnasium-box2d \
ipywidgets notebook lxml fastparquet zstandard \
contextily networkx pyvis netgraph pip
## Project Files
MAIN_SCRIPT := main.py
LOG_FILE := progress.log
## Local Project Paths
LOCAL_PROJECT_DIR := .
LOCAL_OUTPUT_DIR := output
## Remote Project Paths
REMOTE_PROJECT_DIR := ~/my_project
REMOTE_OUTPUT_FILES := output/*
## Misc
COPY_LINKS_AS_FILES ?= false
# ---
LOGIN := $(SERVER_USER)@$(SERVER_HOST)
SSH := ssh $(LOGIN)
RSYNC := rsync --archive --compress --progress $(if $(filter true,$(COPY_LINKS_AS_FILES)),--copy-links)
CONDA := source $(CONDA_PATH)/etc/profile.d/conda.sh && conda
.PHONY: install upload run kill log download delete
help: # list targets and usage
@awk '/^[a-zA-Z0-9_-]+:/{print $1}' makefile
install: # install conda environment
$(SSH) '\
$(CONDA) create -n $(CONDA_ENV) python=$(PYTHON_VERSION) $(PYTHON_PACKAGES)\
'
upload: # upload project files from client to server
$(RSYNC) $(LOCAL_PROJECT_DIR)/ $(LOGIN):$(REMOTE_PROJECT_DIR)
run: # execute main script on server
$(SSH) '\
cd $(REMOTE_PROJECT_DIR);\
$(CONDA) activate $(CONDA_ENV);\
python $(MAIN_SCRIPT);\
$(CONDA) deactivate;\
'
kill: # terminate the running script on the server
$(SSH) '\
pkill -f $(MAIN_SCRIPT)\
'
log: # show the last lines of the log file
$(SSH) '\
cd $(REMOTE_PROJECT_DIR) && tail -f $(LOG_FILE)\
'
download: # download output files from server to client
$(RSYNC) $(LOGIN):$(REMOTE_PROJECT_DIR)/$(REMOTE_OUTPUT_FILES) $(LOCAL_PROJECT_DIR)/$(LOCAL_OUTPUT_DIR)
delete: # remove/delete all project files on server
$(SSH) '\
rm -rf $(REMOTE_PROJECT_DIR)\
'
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment