Skip to content

Commit

Permalink
refactor code to simplify
Browse files Browse the repository at this point in the history
  • Loading branch information
steven-mi committed Aug 15, 2023
1 parent 584738a commit e47a8aa
Showing 1 changed file with 61 additions and 62 deletions.
123 changes: 61 additions & 62 deletions rocket/rocket.py
Original file line number Diff line number Diff line change
Expand Up @@ -103,95 +103,94 @@ def _build_and_deploy(
)
return

if watch:
package_dirs = extract_python_package_dirs(project_location)
files = []
for package_dir in package_dirs:
for file in extract_python_files_from_folder(package_dir):
files.append(file)

project_files = ["setup.py", "pyproject.toml"]
for project_file in project_files:
if os.path.exists(f"{project_location}/{project_file}"):
files.append(f"{project_location}/{project_file}")

if os.path.exists(f"{project_location}/pyproject.toml"):
execute_shell_command(
"poetry export -f requirements.txt --with-credentials --without-hashes --output requirements.txt"
)

dependency_file_exist = False
if not watch:
logger.info(
"Watch is disabled. Building creating a python wheel from your project"
)
wheel_path, wheel_file = self.create_python_project_wheel(project_location)
self._deploy(
file_paths=[wheel_path],
dbfs_path=dbfs_path,
project_location=project_location,
)
install_path = f'{dbfs_path.replace("dbfs:/", "/dbfs/")}/{wheel_file}'

dependency_files = ["requirements.in", "requirements.txt"]
uploaded_dependency_file = ""
index_urls = []
for dependency_file in dependency_files:
dependency_file_path = f"{project_location}/{dependency_file}"
if os.path.exists(dependency_file_path):
files.append(dependency_file_path)
uploaded_dependency_file = dependency_file
dependency_file_exist = True
with open(dependency_file_path) as f:
index_urls = [
line.strip()
for line in f.readlines()
if "index-url" in line
]
self._deploy(
file_paths=files, dbfs_path=dbfs_path, project_location=project_location
index_urls_options = " ".join(index_urls)
logger.info(
f"""Uploaded wheel to databricks. Install your library in your databricks notebook by running:
%pip install --upgrade pip
%pip install {index_urls_options} {install_path} --force-reinstall"""
)
return

install_path = f'{dbfs_path.replace("dbfs:/", "/dbfs/")}'
index_urls_options = " ".join(index_urls)
package_dirs = extract_python_package_dirs(project_location)
files = []
for package_dir in package_dirs:
for file in extract_python_files_from_folder(package_dir):
files.append(file)

if dependency_file_exist:
logger.info(
f"""Watch activated. Uploaded your project to databricks. Install your project in your databricks notebook by running:
project_files = ["setup.py", "pyproject.toml"]
for project_file in project_files:
if os.path.exists(f"{project_location}/{project_file}"):
files.append(f"{project_location}/{project_file}")

if os.path.exists(f"{project_location}/pyproject.toml"):
execute_shell_command(
"poetry export -f requirements.txt --with-credentials --without-hashes --output requirements.txt"
)

dependency_file_exist = False
dependency_files = ["requirements.in", "requirements.txt"]
uploaded_dependency_file = ""
index_urls = []
for dependency_file in dependency_files:
dependency_file_path = f"{project_location}/{dependency_file}"
if os.path.exists(dependency_file_path):
files.append(dependency_file_path)
uploaded_dependency_file = dependency_file
dependency_file_exist = True
with open(dependency_file_path) as f:
index_urls = [
line.strip() for line in f.readlines() if "index-url" in line
]
self._deploy(
file_paths=files, dbfs_path=dbfs_path, project_location=project_location
)

install_path = f'{dbfs_path.replace("dbfs:/", "/dbfs/")}'
index_urls_options = " ".join(index_urls)

if dependency_file_exist:
logger.info(
f"""Watch activated. Uploaded your project to databricks. Install your project in your databricks notebook by running:
%pip install --upgrade pip
%pip install {index_urls_options} -r {install_path}/{uploaded_dependency_file}
%pip install --no-deps -e {install_path}
and following in a new Python cell:
%load_ext autoreload
%autoreload 2"""
)
else:
logger.info(
f"""Watch activated. Uploaded your project to databricks. Install your project in your databricks notebook by running:
)
else:
logger.info(
f"""Watch activated. Uploaded your project to databricks. Install your project in your databricks notebook by running:
%pip install --upgrade pip
%pip install -e {install_path}
and following in a new Python cell:
%load_ext autoreload
%autoreload 2"""
)
else:
logger.info(
"Watch is disabled. Building creating a python wheel from your project"
)
wheel_path, wheel_file = self.create_python_project_wheel(project_location)
self._deploy(
file_paths=[wheel_path],
dbfs_path=dbfs_path,
project_location=project_location,
)
install_path = f'{dbfs_path.replace("dbfs:/", "/dbfs/")}/{wheel_file}'

dependency_files = ["requirements.in", "requirements.txt"]
index_urls = []
for dependency_file in dependency_files:
dependency_file_path = f"{project_location}/{dependency_file}"
if os.path.exists(dependency_file_path):
with open(dependency_file_path) as f:
index_urls = [
line.strip()
for line in f.readlines()
if "index-url" in line
]
index_urls_options = " ".join(index_urls)
logger.info(
f"""Uploaded wheel to databricks. Install your library in your databricks notebook by running:
%pip install --upgrade pip
%pip install {index_urls_options} {install_path} --force-reinstall"""
)

def _deploy(self, file_paths, dbfs_path, project_location):
Expand Down

0 comments on commit e47a8aa

Please sign in to comment.