Skip to content

Relocate plot command line creation to the plotter modules #915

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
altendky merged 2 commits into from
Aug 29, 2021
Merged

Relocate plot command line creation to the plotter modules #915

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ece1247
move plot command line creation to plotter modules
altendky Aug 29, 2021
2d6f2fa
some test coverage
altendky Aug 29, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
109 changes: 108 additions & 1 deletion src/plotman/_tests/plotters/test_init.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ class CommandLineExample:
)


command_line_examples: typing.List[CommandLineExample] = [
chianetwork_command_line_examples: typing.List[CommandLineExample] = [
CommandLineExample(
line=["python", "chia", "plots", "create"],
plotter=plotman.plotters.chianetwork.Plotter,
Expand Down Expand Up @@ -301,6 +301,61 @@ class CommandLineExample:
},
),
),
CommandLineExample(
line=plotman.plotters.chianetwork.create_command_line(
options=plotman.plotters.chianetwork.Options(),
tmpdir="/farm/tmp/dir",
tmp2dir=None,
dstdir="/farm/dst/dir",
farmer_public_key=None,
pool_public_key=None,
pool_contract_address=None,
),
plotter=plotman.plotters.chianetwork.Plotter,
parsed=plotman.job.ParsedChiaPlotsCreateCommand(
error=None,
help=False,
parameters={
**default_chia_network_arguments,
"final_dir": "/farm/dst/dir",
"tmp_dir": "/farm/tmp/dir",
},
),
),
CommandLineExample(
line=plotman.plotters.chianetwork.create_command_line(
options=plotman.plotters.chianetwork.Options(
e=True,
x=True,
),
tmpdir="/farm/tmp/dir",
tmp2dir="/farm/tmp2/dir",
dstdir="/farm/dst/dir",
farmer_public_key="farmerpublickey",
pool_public_key="poolpublickey",
pool_contract_address="poolcontractaddress",
),
plotter=plotman.plotters.chianetwork.Plotter,
parsed=plotman.job.ParsedChiaPlotsCreateCommand(
error=None,
help=False,
parameters={
**default_chia_network_arguments,
"exclude_final_dir": True,
"nobitfield": True,
"farmer_public_key": "farmerpublickey",
"pool_public_key": "poolpublickey",
"pool_contract_address": "poolcontractaddress",
"final_dir": "/farm/dst/dir",
"tmp_dir": "/farm/tmp/dir",
"tmp2_dir": "/farm/tmp2/dir",
},
),
),
]


madmax_command_line_examples: typing.List[CommandLineExample] = [
CommandLineExample(
line=["chia_plot"],
plotter=plotman.plotters.madmax.Plotter,
Expand Down Expand Up @@ -382,6 +437,58 @@ class CommandLineExample:
},
),
),
CommandLineExample(
line=plotman.plotters.madmax.create_command_line(
options=plotman.plotters.madmax.Options(),
tmpdir="/farm/tmp/dir",
tmp2dir=None,
dstdir="/farm/dst/dir",
farmer_public_key=None,
pool_public_key=None,
pool_contract_address=None,
),
plotter=plotman.plotters.madmax.Plotter,
parsed=plotman.job.ParsedChiaPlotsCreateCommand(
error=None,
help=False,
parameters={
**default_madmax_arguments,
"finaldir": pathlib.Path("/farm/dst/dir"),
"tmpdir": pathlib.Path("/farm/tmp/dir"),
},
),
),
CommandLineExample(
line=plotman.plotters.madmax.create_command_line(
options=plotman.plotters.madmax.Options(),
tmpdir="/farm/tmp/dir",
tmp2dir="/farm/tmp2/dir",
dstdir="/farm/dst/dir",
farmer_public_key="farmerpublickey",
pool_public_key="poolpublickey",
pool_contract_address="poolcontractaddress",
),
plotter=plotman.plotters.madmax.Plotter,
parsed=plotman.job.ParsedChiaPlotsCreateCommand(
error=None,
help=False,
parameters={
**default_madmax_arguments,
"farmerkey": "farmerpublickey",
"poolkey": "poolpublickey",
"contract": "poolcontractaddress",
"finaldir": pathlib.Path("/farm/dst/dir"),
"tmpdir": pathlib.Path("/farm/tmp/dir"),
"tmpdir2": pathlib.Path("/farm/tmp2/dir"),
},
),
),
]


command_line_examples: typing.List[CommandLineExample] = [
*chianetwork_command_line_examples,
*madmax_command_line_examples,
]

not_command_line_examples: typing.List[CommandLineExample] = [
Expand Down
63 changes: 13 additions & 50 deletions src/plotman/configuration.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
import importlib
import os
import stat
import subprocess
import tempfile
import textwrap
from typing import Dict, Generator, List, Mapping, Optional
Expand All @@ -16,11 +15,12 @@
import marshmallow
import marshmallow.fields
import marshmallow.validate
import packaging.version
import pendulum
import yaml

from plotman import resources as plotman_resources
import plotman.plotters.chianetwork
import plotman.plotters.madmax


class ConfigurationException(Exception):
Expand Down Expand Up @@ -373,26 +373,6 @@ class Scheduling:
tmp_overrides: Optional[Dict[str, TmpOverrides]] = None


@attr.frozen
class ChiaPlotterOptions:
executable: str = "chia"
n_threads: int = 2
n_buckets: int = 128
k: Optional[int] = 32
e: Optional[bool] = False
job_buffer: Optional[int] = 3389
x: bool = False


@attr.frozen
class MadmaxPlotterOptions:
executable: str = "chia_plot"
n_threads: int = 4
n_buckets: int = 256
n_buckets3: int = 256
n_rmulti2: int = 1


@attr.frozen
class Plotting:
farmer_pk: Optional[str] = None
Expand All @@ -408,8 +388,8 @@ class Plotting:
},
},
)
chia: Optional[ChiaPlotterOptions] = None
madmax: Optional[MadmaxPlotterOptions] = None
chia: Optional[plotman.plotters.chianetwork.Options] = None
madmax: Optional[plotman.plotters.madmax.Options] = None


@attr.frozen
Expand Down Expand Up @@ -448,20 +428,11 @@ def setup(self) -> Generator[None, None, None]:
+ " full configuration file"
)
raise Exception(message)
if self.plotting.pool_contract_address is not None:
completed_process = subprocess.run(
args=[self.plotting.chia.executable, "version"],
capture_output=True,
check=True,
encoding="utf-8",
)
version = packaging.version.Version(completed_process.stdout)
required_version = packaging.version.Version("1.2")
if version < required_version:
raise Exception(
f"Chia version {required_version} required for creating pool"
f" plots but found: {version}"
)

plotman.plotters.chianetwork.check_configuration(
options=self.plotting.chia,
pool_contract_address=self.plotting.pool_contract_address,
)
elif self.plotting.type == "madmax":
if self.plotting.madmax is None:
message = (
Expand All @@ -470,18 +441,10 @@ def setup(self) -> Generator[None, None, None]:
)
raise Exception(message)

if self.plotting.pool_contract_address is not None:
completed_process = subprocess.run(
args=[self.plotting.madmax.executable, "--help"],
capture_output=True,
check=True,
encoding="utf-8",
)
if "--contract" not in completed_process.stdout:
raise Exception(
f"found madMAx version does not support the `--contract`"
f" option for pools."
)
plotman.plotters.madmax.check_configuration(
options=self.plotting.madmax,
pool_contract_address=self.plotting.pool_contract_address,
)

prefix = f"plotman-pid_{os.getpid()}-"

Expand Down
80 changes: 21 additions & 59 deletions src/plotman/manager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@
) # for get_archdir_freebytes(). TODO: move to avoid import loop
from plotman import job, plot_util
import plotman.configuration
import plotman.plotters.chianetwork
import plotman.plotters.madmax


# Constants
MIN = 60 # Seconds
Expand Down Expand Up @@ -186,70 +189,29 @@ def key(key: str) -> job.Phase:
raise Exception(
"madmax plotter selected but not configured, report this as a plotman bug",
)
plot_args = [
plotting_cfg.madmax.executable,
"-n",
str(1),
"-r",
str(plotting_cfg.madmax.n_threads),
"-u",
str(plotting_cfg.madmax.n_buckets),
"-t",
tmpdir if tmpdir.endswith("/") else (tmpdir + "/"),
"-d",
dstdir if dstdir.endswith("/") else (dstdir + "/"),
]
if dir_cfg.tmp2 is not None:
plot_args.append("-2")
plot_args.append(
dir_cfg.tmp2
if dir_cfg.tmp2.endswith("/")
else (dir_cfg.tmp2 + "/")
)
if plotting_cfg.madmax.n_buckets3 is not None:
plot_args.append("-v")
plot_args.append(str(plotting_cfg.madmax.n_buckets3))
if plotting_cfg.madmax.n_rmulti2 is not None:
plot_args.append("-K")
plot_args.append(str(plotting_cfg.madmax.n_rmulti2))
plot_args = plotman.plotters.madmax.create_command_line(
options=plotting_cfg.madmax,
tmpdir=tmpdir,
tmp2dir=dir_cfg.tmp2,
dstdir=dstdir,
farmer_public_key=plotting_cfg.farmer_pk,
pool_public_key=plotting_cfg.pool_pk,
pool_contract_address=plotting_cfg.pool_contract_address,
)
else:
if plotting_cfg.chia is None:
raise Exception(
"chia plotter selected but not configured, report this as a plotman bug",
)
plot_args = [
plotting_cfg.chia.executable,
"plots",
"create",
"-k",
str(plotting_cfg.chia.k),
"-r",
str(plotting_cfg.chia.n_threads),
"-u",
str(plotting_cfg.chia.n_buckets),
"-b",
str(plotting_cfg.chia.job_buffer),
"-t",
tmpdir,
"-d",
dstdir,
]
if plotting_cfg.chia.e:
plot_args.append("-e")
if plotting_cfg.chia.x:
plot_args.append("-x")
if dir_cfg.tmp2 is not None:
plot_args.append("-2")
plot_args.append(dir_cfg.tmp2)
if plotting_cfg.farmer_pk is not None:
plot_args.append("-f")
plot_args.append(plotting_cfg.farmer_pk)
if plotting_cfg.pool_pk is not None:
plot_args.append("-p")
plot_args.append(plotting_cfg.pool_pk)
if plotting_cfg.pool_contract_address is not None:
plot_args.append("-c")
plot_args.append(plotting_cfg.pool_contract_address)
plot_args = plotman.plotters.chianetwork.create_command_line(
options=plotting_cfg.chia,
tmpdir=tmpdir,
tmp2dir=dir_cfg.tmp2,
dstdir=dstdir,
farmer_public_key=plotting_cfg.farmer_pk,
pool_public_key=plotting_cfg.pool_pk,
pool_contract_address=plotting_cfg.pool_contract_address,
)

logmsg = "Starting plot job: %s ; logging to %s" % (
" ".join(plot_args),
Expand Down
Loading