ContainerRecipe
A dict-like representation of container's recipe
The recipe file of default container '.main' will be '~/rezup.toml'
And for other container, it will be ~/rezup.{name}.toml
for example:
* ~/rezup.dev.toml -> container dev
* ~/rezup.test.toml -> container test
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
name |
str |
Container name |
None |
Source code in rezup/recipe.py
class ContainerRecipe(BaseRecipe):
"""A dict-like representation of container's recipe
The recipe file of default container '.main' will be '~/rezup.toml'
And for other container, it will be `~/rezup.{name}.toml`
for example:
* `~/rezup.dev.toml` -> container `dev`
* `~/rezup.test.toml` -> container `test`
Args:
name (str, optional): Container name
"""
REGEX = re.compile("rezup.?(.*).toml")
RECIPES_DIR = DEFAULT_CONTAINER_RECIPES #: `DEFAULT_CONTAINER_RECIPES`
def __init__(self, name=None):
super(ContainerRecipe, self).__init__(name)
_name = self._name
if _name and _name != DEFAULT_CONTAINER_NAME:
self._file = "rezup.%s.toml" % _name
else:
self._file = "rezup.toml"
@classmethod
@contextmanager
def provisional_recipes(cls, path):
"""Context for changing recipes root temporarily
Container recipes should be in user's home directory by defaule, but
that could be changed inside this context for the case when you need
to operate on machines that have no recipe exists in home directory.
```
with ContainerRecipe.provisional_recipes("/to/other/recipes"):
...
```
Args:
path (str or path-like): directory path where recipes located
"""
default = cls.RECIPES_DIR
try:
cls.RECIPES_DIR = Path(path)
yield
finally:
cls.RECIPES_DIR = default
@classmethod
def iter_recipes(cls):
"""Iter all recipe files found in `ContainerRecipe.RECIPES_DIR`
Yields:
`ContainerRecipe`
"""
for item in cls.RECIPES_DIR.iterdir():
if not item.is_file():
continue
match = cls.REGEX.search(item.name)
if match:
name = match.group(1)
yield cls(name)
def path(self):
"""Returns the file path of this recipe
Returns:
pathlib.Path: The file path of this recipe
"""
if self._path is None:
self._path = self.RECIPES_DIR / self._file
return self._path
def create(self, data=None):
"""Write out recipe content into a .toml file
Args:
data (dict, optional): Arbitrary data to write out
"""
if not self.RECIPES_DIR.is_dir():
self.RECIPES_DIR.mkdir(parents=True)
path = self.path()
if data:
_data = toml.load(str(self.DEFAULT_RECIPE))
deep_update(_data, data)
with open(str(path), "w") as f:
toml.dump(_data, f)
else:
# read & write as plaintext, so the comment can be preserved
with open(str(self.DEFAULT_RECIPE), "r") as r:
with open(str(path), "w") as w:
w.write(r.read())
self._load()
create(self, data=None)
Write out recipe content into a .toml file
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
data |
dict |
Arbitrary data to write out |
None |
Source code in rezup/recipe.py
def create(self, data=None):
"""Write out recipe content into a .toml file
Args:
data (dict, optional): Arbitrary data to write out
"""
if not self.RECIPES_DIR.is_dir():
self.RECIPES_DIR.mkdir(parents=True)
path = self.path()
if data:
_data = toml.load(str(self.DEFAULT_RECIPE))
deep_update(_data, data)
with open(str(path), "w") as f:
toml.dump(_data, f)
else:
# read & write as plaintext, so the comment can be preserved
with open(str(self.DEFAULT_RECIPE), "r") as r:
with open(str(path), "w") as w:
w.write(r.read())
self._load()
iter_recipes()
classmethod
Iter all recipe files found in ContainerRecipe.RECIPES_DIR
Yields:
| Type | Description |
|---|---|
|
Source code in rezup/recipe.py
@classmethod
def iter_recipes(cls):
"""Iter all recipe files found in `ContainerRecipe.RECIPES_DIR`
Yields:
`ContainerRecipe`
"""
for item in cls.RECIPES_DIR.iterdir():
if not item.is_file():
continue
match = cls.REGEX.search(item.name)
if match:
name = match.group(1)
yield cls(name)
path(self)
Returns the file path of this recipe
Returns:
| Type | Description |
|---|---|
pathlib.Path |
The file path of this recipe |
Source code in rezup/recipe.py
def path(self):
"""Returns the file path of this recipe
Returns:
pathlib.Path: The file path of this recipe
"""
if self._path is None:
self._path = self.RECIPES_DIR / self._file
return self._path
provisional_recipes(cls, path)
classmethod
Context for changing recipes root temporarily
Container recipes should be in user's home directory by defaule, but that could be changed inside this context for the case when you need to operate on machines that have no recipe exists in home directory.
with ContainerRecipe.provisional_recipes("/to/other/recipes"):
...
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
path |
str or path-like |
directory path where recipes located |
required |
Source code in rezup/recipe.py
@classmethod
@contextmanager
def provisional_recipes(cls, path):
"""Context for changing recipes root temporarily
Container recipes should be in user's home directory by defaule, but
that could be changed inside this context for the case when you need
to operate on machines that have no recipe exists in home directory.
```
with ContainerRecipe.provisional_recipes("/to/other/recipes"):
...
```
Args:
path (str or path-like): directory path where recipes located
"""
default = cls.RECIPES_DIR
try:
cls.RECIPES_DIR = Path(path)
yield
finally:
cls.RECIPES_DIR = default
get_container_root
Internal use
Get container root path from recipe
# rezup.foo.toml
[root]
local = ""
remote = "/path/to/remote/containers"
If the recipe represents the above example recipe file, container named
foo will have local root set to rezup.recipe.DEFAULT_CONTAINER_RECIPES,
unless environ variable REZUP_ROOT_LOCAL has value. And the remote root
will set to /path/to/remote/containers.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
recipe |
`ContainerRecipe` |
The container recipe to lookup from. |
required |
remote |
bool |
Fetch remote root if True, otherwise local. |
False |
Returns:
| Type | Description |
|---|---|
`pathlib.Path` |
Container root path, local or remote one. |
Source code in rezup/container.py
def get_container_root(recipe, remote=False):
"""Get container root path from recipe
```toml
# rezup.foo.toml
[root]
local = ""
remote = "/path/to/remote/containers"
```
If the `recipe` represents the above example recipe file, container named
`foo` will have local root set to `rezup.recipe.DEFAULT_CONTAINER_RECIPES`,
unless environ variable `REZUP_ROOT_LOCAL` has value. And the remote root
will set to `/path/to/remote/containers`.
Args:
recipe (`ContainerRecipe`): The container recipe to lookup from.
remote (bool): Fetch remote root if True, otherwise local.
Returns:
`pathlib.Path`: Container root path, local or remote one.
"""
data = recipe.data()
if remote:
remote = \
data["root"]["remote"] \
or os.getenv("REZUP_ROOT_REMOTE")
return Path(norm_path(remote)) if remote else None
else:
local = \
data["root"]["local"] \
or os.getenv("REZUP_ROOT_LOCAL") \
or "~/.rezup"
return Path(norm_path(local))
Container
Timestamp ordered virtual environment stack
A Rez venv provider, that allows updating venv continuously without affecting any existing consumer.
In filesystem, a container is a folder that has at least one Rez venv
installation exists, and those Rez venv folders (Revision) are named
by timestamp, so when the container is being asked for a venv to use,
the current latest available one will be sorted out.
The location of the container can be set with env var REZUP_ROOT_LOCAL
or ~/.rezup will be used by default.
For centralize management in production, one remote container can be
defined with an env var REZUP_ROOT_REMOTE. The remote container only
contains the venv installation manifest file, and when being asked, a
venv will be created locally or re-used if same revision exists in local.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
name |
`str` |
Container name, use |
None |
recipe |
`ContainerRecipe` |
A |
None |
force_local |
`bool` |
Default |
False |
Source code in rezup/container.py
class Container:
"""Timestamp ordered virtual environment stack
A Rez venv provider, that allows updating venv continuously without
affecting any existing consumer.
In filesystem, a container is a folder that has at least one Rez venv
installation exists, and those Rez venv folders (`Revision`) are named
by timestamp, so when the container is being asked for a venv to use,
the current latest available one will be sorted out.
The location of the container can be set with env var `REZUP_ROOT_LOCAL`
or `~/.rezup` will be used by default.
For centralize management in production, one remote container can be
defined with an env var `REZUP_ROOT_REMOTE`. The remote container only
contains the venv installation manifest file, and when being asked, a
venv will be created locally or re-used if same revision exists in local.
Args:
name (`str`, optional): Container name, use `Container.DEFAULT_NAME`
if not given.
recipe (`ContainerRecipe`, optional): A `ContainerRecipe` object to
help constructing container, will look into `ContainerRecipe.RECIPES_DIR`
if not given.
force_local (`bool`, optional): Default `False`. Ignore linking remote
even if the remote path has set, always link to local if `True`.
"""
DEFAULT_NAME = DEFAULT_CONTAINER_NAME #: `rezup.recipe.DEFAULT_CONTAINER_NAME`
def __init__(self, name=None, recipe=None, force_local=False):
name = name or self.DEFAULT_NAME
recipe = recipe or ContainerRecipe(name)
local_root = get_container_root(recipe, remote=False)
remote_root = get_container_root(recipe, remote=True)
if force_local:
root = local_root
else:
root = remote_root or local_root
self._remote = root == remote_root
self._recipe = recipe
self._name = name
self._root = root
self._path = root / name
def __repr__(self):
return "%s(remote=%d, name=%r, path=%r)" % (
self.__class__.__name__,
int(self.is_remote()),
self.name(),
self._path.resolve(),
)
@classmethod
def create(cls, name, force_local=False):
"""Create container from recipe.
Args:
name (str): The name of container.
force_local (bool): Create as local if True, or try remote.
Returns:
`Container`: A container instance.
"""
recipe = ContainerRecipe(name)
_log.debug("Sourcing recipe from: %s" % recipe)
if not recipe.is_file():
_log.debug("Recipe file not exists, creating default..")
recipe.create()
return Container(name, recipe, force_local)
def root(self):
"""
Returns:
`pathlib.Path`: Path to where this container is located.
"""
return self._root
def name(self):
"""
Returns:
str: Container name.
"""
return self._name
def path(self):
"""
Returns:
`pathlib.Path`: Full path of this container.
"""
return self._path
def recipe(self):
"""
Returns:
`ContainerRecipe`: The recipe instance that binds to this container.
"""
return self._recipe
def libs(self):
"""
Returns:
`pathlib.Path`: Path to this container's shared libraries.
"""
return self._path / "libs"
def revisions(self):
"""
Returns:
`pathlib.Path`: Path to this container's revisions.
"""
return self._path / "revisions"
def is_exists(self):
"""
Returns:
bool: Return `True` if this container exists.
"""
return self._path.is_dir()
def is_empty(self):
"""
Returns:
bool: Return `True` if this container has no valid revisions.
"""
return not bool(next(self.iter_revision(), None))
def is_remote(self):
"""
Returns:
bool: Return `True` if this container is linked to remote side.
"""
return self._remote
def purge(self):
if self.is_exists():
revision = self.get_latest_revision(only_ready=False)
if not revision:
# TODO: don't remove it immediately, mark as purged and
# remove it when $REZUP_CLEAN_AFTER meet
rmtree(self._path)
else:
# TODO: should have better exception type
# TODO: need to check revision is in use
raise Exception("Revision is creating.")
# keep tidy, try remove the root of containers if it's now empty
root = self.root()
if root.is_dir() and not next(root.iterdir(), None):
rmtree(root)
def iter_revision(self, validate=True, latest_first=True):
"""Iterating revisions from this container.
Args:
validate (bool, optional): Default `True`. Only yield revisions
that are valid if `True`.
latest_first (bool): Default `True`. Yield revisions based on
it's directory name (timestamp string) in descending order,
or ascending if `False`.
Yields:
Revision: `Revision` instances that match the condition.
"""
_log.debug("Iterating revisions in container %s.." % self)
if not self.is_exists():
_log.debug("Container %r not exists." % self.name())
return
revisions_root = Revision.compose_path(container=self)
if not revisions_root.is_dir():
_log.debug("Revision root is not a directory: %s" % revisions_root)
return
revisions_root = str(revisions_root)
for entry in sorted(os.listdir(revisions_root), reverse=latest_first):
revision = Revision(container=self, dirname=entry)
_log.debug("... %s" % revision)
if not validate or revision.is_valid():
yield revision
def get_latest_revision(self, only_ready=True):
"""Get latest revision from this container.
Args:
only_ready (bool): Default `True`. Include revisions that are not
in ready state if `False`.
Returns:
Revision: An instance of `Revision` if found, or `None`.
"""
for revision in self.iter_revision():
if not only_ready or revision.is_ready():
_log.debug("Found latest revision.")
return revision
def get_revision_by_time(self, timestamp, fallback=False, only_ready=True):
"""Returns a revision that match the timestamp
Args:
timestamp (datetime.datetime): a time for matching revision
fallback (bool): If True, accept earlier revision when no exact matched
only_ready (bool): Include revisions that are not in ready state if False
Returns:
Revision: An instance of `Revision` if found, or `None`.
"""
for revision in self.iter_revision():
if not only_ready or revision.is_ready():
if (revision.timestamp() == timestamp
or (fallback and revision.timestamp() <= timestamp)):
_log.debug("Found time matched revision.")
return revision
_log.debug("No time matched revision found.")
def new_revision(self):
"""Create a new revision
Returns:
Revision: An instance of `Revision` that just created.
"""
return Revision.create(self)
create(name, force_local=False)
classmethod
Create container from recipe.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
name |
str |
The name of container. |
required |
force_local |
bool |
Create as local if True, or try remote. |
False |
Returns:
| Type | Description |
|---|---|
`Container` |
A container instance. |
Source code in rezup/container.py
@classmethod
def create(cls, name, force_local=False):
"""Create container from recipe.
Args:
name (str): The name of container.
force_local (bool): Create as local if True, or try remote.
Returns:
`Container`: A container instance.
"""
recipe = ContainerRecipe(name)
_log.debug("Sourcing recipe from: %s" % recipe)
if not recipe.is_file():
_log.debug("Recipe file not exists, creating default..")
recipe.create()
return Container(name, recipe, force_local)
get_latest_revision(self, only_ready=True)
Get latest revision from this container.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
only_ready |
bool |
Default |
True |
Returns:
| Type | Description |
|---|---|
Revision |
An instance of |
Source code in rezup/container.py
def get_latest_revision(self, only_ready=True):
"""Get latest revision from this container.
Args:
only_ready (bool): Default `True`. Include revisions that are not
in ready state if `False`.
Returns:
Revision: An instance of `Revision` if found, or `None`.
"""
for revision in self.iter_revision():
if not only_ready or revision.is_ready():
_log.debug("Found latest revision.")
return revision
get_revision_by_time(self, timestamp, fallback=False, only_ready=True)
Returns a revision that match the timestamp
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
timestamp |
datetime.datetime |
a time for matching revision |
required |
fallback |
bool |
If True, accept earlier revision when no exact matched |
False |
only_ready |
bool |
Include revisions that are not in ready state if False |
True |
Returns:
| Type | Description |
|---|---|
Revision |
An instance of |
Source code in rezup/container.py
def get_revision_by_time(self, timestamp, fallback=False, only_ready=True):
"""Returns a revision that match the timestamp
Args:
timestamp (datetime.datetime): a time for matching revision
fallback (bool): If True, accept earlier revision when no exact matched
only_ready (bool): Include revisions that are not in ready state if False
Returns:
Revision: An instance of `Revision` if found, or `None`.
"""
for revision in self.iter_revision():
if not only_ready or revision.is_ready():
if (revision.timestamp() == timestamp
or (fallback and revision.timestamp() <= timestamp)):
_log.debug("Found time matched revision.")
return revision
_log.debug("No time matched revision found.")
is_empty(self)
Returns:
| Type | Description |
|---|---|
bool |
Return |
Source code in rezup/container.py
def is_empty(self):
"""
Returns:
bool: Return `True` if this container has no valid revisions.
"""
return not bool(next(self.iter_revision(), None))
is_exists(self)
Returns:
| Type | Description |
|---|---|
bool |
Return |
Source code in rezup/container.py
def is_exists(self):
"""
Returns:
bool: Return `True` if this container exists.
"""
return self._path.is_dir()
is_remote(self)
Returns:
| Type | Description |
|---|---|
bool |
Return |
Source code in rezup/container.py
def is_remote(self):
"""
Returns:
bool: Return `True` if this container is linked to remote side.
"""
return self._remote
iter_revision(self, validate=True, latest_first=True)
Iterating revisions from this container.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
validate |
bool |
Default |
True |
latest_first |
bool |
Default |
True |
Yields:
| Type | Description |
|---|---|
Revision |
|
Source code in rezup/container.py
def iter_revision(self, validate=True, latest_first=True):
"""Iterating revisions from this container.
Args:
validate (bool, optional): Default `True`. Only yield revisions
that are valid if `True`.
latest_first (bool): Default `True`. Yield revisions based on
it's directory name (timestamp string) in descending order,
or ascending if `False`.
Yields:
Revision: `Revision` instances that match the condition.
"""
_log.debug("Iterating revisions in container %s.." % self)
if not self.is_exists():
_log.debug("Container %r not exists." % self.name())
return
revisions_root = Revision.compose_path(container=self)
if not revisions_root.is_dir():
_log.debug("Revision root is not a directory: %s" % revisions_root)
return
revisions_root = str(revisions_root)
for entry in sorted(os.listdir(revisions_root), reverse=latest_first):
revision = Revision(container=self, dirname=entry)
_log.debug("... %s" % revision)
if not validate or revision.is_valid():
yield revision
libs(self)
Returns:
| Type | Description |
|---|---|
`pathlib.Path` |
Path to this container's shared libraries. |
Source code in rezup/container.py
def libs(self):
"""
Returns:
`pathlib.Path`: Path to this container's shared libraries.
"""
return self._path / "libs"
name(self)
Returns:
| Type | Description |
|---|---|
str |
Container name. |
Source code in rezup/container.py
def name(self):
"""
Returns:
str: Container name.
"""
return self._name
new_revision(self)
Create a new revision
Returns:
| Type | Description |
|---|---|
Revision |
An instance of |
Source code in rezup/container.py
def new_revision(self):
"""Create a new revision
Returns:
Revision: An instance of `Revision` that just created.
"""
return Revision.create(self)
path(self)
Returns:
| Type | Description |
|---|---|
`pathlib.Path` |
Full path of this container. |
Source code in rezup/container.py
def path(self):
"""
Returns:
`pathlib.Path`: Full path of this container.
"""
return self._path
recipe(self)
Returns:
| Type | Description |
|---|---|
`ContainerRecipe` |
The recipe instance that binds to this container. |
Source code in rezup/container.py
def recipe(self):
"""
Returns:
`ContainerRecipe`: The recipe instance that binds to this container.
"""
return self._recipe
revisions(self)
Returns:
| Type | Description |
|---|---|
`pathlib.Path` |
Path to this container's revisions. |
Source code in rezup/container.py
def revisions(self):
"""
Returns:
`pathlib.Path`: Path to this container's revisions.
"""
return self._path / "revisions"
root(self)
Returns:
| Type | Description |
|---|---|
`pathlib.Path` |
Path to where this container is located. |
Source code in rezup/container.py
def root(self):
"""
Returns:
`pathlib.Path`: Path to where this container is located.
"""
return self._root
Revision
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
container |
Container |
The container that holds this revision |
required |
dirname |
str |
Directory name of this revision, should be a timestamp string if given. |
None |
Source code in rezup/container.py
class Revision:
"""
Args:
container (Container): The container that holds this revision
dirname (str, optional): Directory name of this revision, should be
a timestamp string if given.
"""
def __init__(self, container, dirname=None):
dirname = str(dirname or time.time())
self._container = container
self._dirname = dirname
self._path = self.compose_path(container, dirname)
self._timestamp = None
self._is_valid = None
self._metadata = None
self._recipe = RevisionRecipe(self)
self._metadata_path = self._path / "revision.json"
self._is_pulled = False
def __repr__(self):
return "%s(valid=%d, ready=%d, remote=%d, time=%s, path=%r)" % (
self.__class__.__name__,
int(self.is_valid()),
int(self.is_ready()),
int(self.is_remote()),
self.time_str() or "?",
self._path.resolve(),
)
def __eq__(self, other):
if not isinstance(other, type(self)):
return False
return self._container.name() == other._container.name() \
and self.timestamp() == other.timestamp()
@classmethod
def compose_path(cls, container, dirname=None):
path = container.revisions()
if dirname:
path /= dirname
return path
@classmethod
def create(cls, container):
revision = cls(container=container)
revision._write()
return revision
def _write(self, pulling=None):
"""Sourcing recipe and create a revision
Args:
pulling (Revision, optional): If given, pulling recipe from that
revision, usually a remote one.
"""
_log.info("Creating revision..")
recipe = self._recipe
makedirs(self._path)
# write revision recipe
if pulling is None:
_log.info("Recipe sourced from: %s" % self._container.recipe())
recipe.create()
else:
_log.debug("Pulling recipe from: %s" % pulling)
recipe.pull(pulling)
if not self.is_valid():
raise Exception("Invalid new revision, this is a bug.")
# manifest
rez_ = Tool(recipe["rez"])
extensions = [Tool(d) for d in recipe.get("extension", []) if d]
shared_lib = recipe.get("shared")
_log.info("Recipe loaded.")
_log.debug(" Rez: %s" % rez_.url)
_log.debug(" Extension: " + ", ".join([e.name for e in extensions]))
_log.debug("Shared-lib: %s" % shared_lib)
# install, if at local
if not self._container.is_remote():
self._install(rez_, extensions, shared_lib)
# save metadata, mark revision as ready
with open(str(self._metadata_path), "w") as f:
# metadata
f.write(json.dumps({
"rezup_version": __version__,
"creator": getpass.getuser(),
"hostname": socket.gethostname(),
"revision_path": str(self._path),
"venvs": ["rez"] + [t.name for t in extensions if t.isolation],
}, indent=4))
_log.info("Revision created: %s" % self)
def _install(self, rez_, extensions=None, shared_lib=None):
"""Construct Rez virtual environment by recipe
"""
_log.debug("Installing..")
pip_entry = self._recipe.get("pip")
extensions = extensions or []
installer = Installer(
self,
pip_opt=pip_entry.get("options"),
pip_env=pip_entry.get("env"),
)
installer.install_rez(rez_)
if shared_lib:
warnings.warn("Shared-lib section is about to be deprecated, "
"use 'rez.lib' or 'extension.lib' section instead. "
"See https://github.com/davidlatwe/rezup/issues/62",
DeprecationWarning)
installer.create_shared_lib(name=shared_lib["name"],
requires=shared_lib["requires"])
for ext in extensions:
installer.install_extension(ext)
def validate(self):
is_valid = True
seconds = float(self._dirname)
timestamp = datetime.fromtimestamp(seconds)
if self._recipe.is_file():
self._timestamp = timestamp
else:
is_valid = False
return is_valid
def is_valid(self):
if self._is_valid is None:
try:
self._is_valid = self.validate()
except (OSError, ValueError):
self._is_valid = False
return self._is_valid
def is_ready(self):
return self._metadata_path.is_file()
def is_remote(self):
return self._container.is_remote()
def dirname(self):
return self._dirname
def path(self):
return self._path
def timestamp(self):
return self._timestamp
def time_str(self):
return self._timestamp.strftime("%d.%b.%y %H:%M") \
if self._timestamp else ""
def container(self):
return self._container
def metadata(self):
if self.is_ready():
if self._metadata is None:
with open(str(self._metadata_path), "r") as f:
self._metadata = json.load(f)
return self._metadata
def recipe(self):
if self.is_valid():
return self._recipe
def recipe_env(self):
_platform = platform.system().lower()
recipe = self.recipe() or {}
env = {}
def load_env(**kwargs):
return {
k: v for k, v in dotenv_values(**kwargs).items()
if v is not None # exclude config section line
}
def file_loader(d):
return [d[k] for k in sorted([
k for k, v in d.items() if isinstance(v, string_types)])]
dot_env = recipe.get("dotenv")
if dot_env:
# non platform specific dotenv
env_files = file_loader(dot_env)
if isinstance(dot_env.get(_platform), dict):
# platform specific dotenv
env_files += file_loader(dot_env[_platform])
for file in env_files:
env.update(load_env(dotenv_path=file))
recipe_env = recipe.get("env")
if recipe_env:
stream = StringIO()
_parser = ConfigParser()
_parser.optionxform = str # to prevent turning keys into lowercase
_parser.read_dict({"env": recipe_env})
_parser.write(stream)
stream.seek(0) # must reset buffer
env.update(load_env(stream=stream))
env.update({
"REZUP_CONTAINER": self._container.name(),
"REZUP_USING_REMOTE": "yes" if self._is_pulled else "",
})
return env
def purge(self):
if self.is_valid():
# TODO: don't remove it immediately, mark as purged and
# remove it when $REZUP_CLEAN_AFTER meet
rmtree(self._path)
def iter_backward(self):
for revision in self._container.iter_revision(latest_first=True):
if revision.timestamp() < self._timestamp:
yield revision
def iter_forward(self):
for revision in self._container.iter_revision(latest_first=False):
if revision.timestamp() > self._timestamp:
yield revision
def pull(self, check_out=True, fallback=False):
"""Return corresponding local side revision
If the revision is from remote container, calling this method will
find a timestamp matched local revision and create one if not found
by default.
If the revision is from local container, return `self`.
Args:
check_out (bool, optional): When no matched local revision,
create one if True or just return None at the end.
Default is True.
fallback (bool, optional)
Returns:
Revision or None
"""
if not self.is_remote():
return self
# get local
_con_name = self._container.name()
_con_recipe = self._container.recipe() # careful, this affect's root
local = Container(_con_name, recipe=_con_recipe, force_local=True)
rev = local.get_revision_by_time(self._timestamp,
fallback=fallback,
only_ready=True)
_allow_create = rev is None and check_out
_did_fallback = rev.timestamp() != self._timestamp if rev else False
if not fallback and _allow_create:
_log.info("Pulling from remote container: %s" % self._container)
rev = Revision(container=local, dirname=self._dirname)
rev._write(pulling=self)
if fallback and _did_fallback:
_log.warning(
"Local revision at exact time (%s) is not ready, "
"fallback to %s" % (self.time_str(), rev)
)
if rev is not None:
rev._is_pulled = True
return rev
def spawn_shell(self, command=None):
"""Spawn a sub-shell
Args:
command (list, optional): Shell script file with args or commands.
If given, the sub-shell will not be interactive.
Returns:
subprocess.Popen
Raises:
ContainerError
"""
if not self.is_valid():
raise ContainerError("Cannot use invalid revision.")
if not self.is_ready():
raise ContainerError("Revision is not ready to be used.")
if self.is_remote():
# use local
revision = self.pull()
if revision is None:
raise ContainerError("No revision pulled.")
if not revision.is_ready():
raise ContainerError("Revision is not ready to be used.")
return revision.spawn_shell(command=command)
else:
# Launch subprocess
environment = self._compose_env()
shell_name, shell_exec = self._get_shell()
if command:
# run command and exit
if command[0] == ".":
cmd = command
else:
exe = command[0]
exe = shell.which(exe, env=environment) or exe
cmd = [exe] + command[1:]
else:
# interactive shell
_con_name = self._container.name()
_con_from = "remote" if self._is_pulled else "local"
prompt = "rezup (%s/%s) " % (_con_name, _con_from)
prompt = shell.format_prompt_code(prompt, shell_name)
environment.update({
"REZUP_PROMPT": os.getenv("REZUP_PROMPT", prompt),
})
cmd = shell.get_launch_cmd(
shell_name,
shell_exec,
interactive=True,
)
popen = subprocess.Popen(cmd, env=environment)
return popen
def use(self, command=None, wait=True):
"""Run a sub-shell
Args:
command (list, optional): Shell script with args or commands. If
given, the sub-shell will not be interactive.
wait (bool, optional): Whether to wait `command` finish or not,
default True.
Returns:
int: subprocess return code, will always return 0 if `command`
is given and `wait` is False.
"""
block = not command
popen = self.spawn_shell(command=command)
if block or wait:
stdout, stderr = popen.communicate()
return popen.returncode
else:
return 0
def _compose_env(self):
env = os.environ.copy()
env.update(self.recipe_env() or {})
env["PATH"] = os.pathsep.join([
os.pathsep.join([str(p) for p in self.production_bin_dirs()]),
env["PATH"]
])
# use `pythonfinder` package if need to exclude python from PATH
return env
def _get_shell(self):
shell_name = os.getenv("REZUP_DEFAULT_SHELL")
if shell_name:
shell_exec = shell_name
shell_name, ext = os.path.splitext(os.path.basename(shell_name))
shell_name = shell_name.lower()
else:
shell_name, shell_exec = shell.get_current_shell()
shell_exec = shell_exec or shell_name
return shell_name, shell_exec
def _require_local(method): # noqa
"""Decorator for ensuring local revision exists before action
"""
@functools.wraps(method) # noqa
def wrapper(self, *args, **kwargs):
if not self.is_remote():
return method(self, *args, **kwargs) # noqa
check_out = bool(os.getenv("REZUP_ALWAYS_CHECKOUT"))
revision = self.pull(check_out=check_out)
if revision is None:
raise ContainerError(
"This revision is from remote container, no matched found "
"in local. Possible not been pulled into local yet."
)
return getattr(revision, method.__name__)(self, *args, **kwargs)
return wrapper
@_require_local # noqa
def locate_rez_lib(self, venv_session=None):
"""Returns rez module location in this revision
Returns:
pathlib.Path or None if not found.
"""
if venv_session is None:
venv_path = self.path() / "venv" / "rez"
venv_session = virtualenv.session_via_cli(args=[str(venv_path)])
venv_lib = venv_session.creator.purelib
# rez may get installed in edit mode, try short route first
egg_link = venv_lib / "rez.egg-link"
if egg_link.is_file():
with open(str(egg_link), "r") as f:
package_location = f.readline().strip()
if os.path.isdir(package_location):
return Path(package_location)
for importer, modname, pkg in pkgutil.walk_packages([str(venv_lib)]):
if pkg and modname == "rez":
loader = importer.find_module(modname)
try:
path = loader.path # SourceFileLoader
return Path(path).parent.parent
except AttributeError:
path = loader.filename # ImpLoader, py2
return Path(path).parent
@_require_local # noqa
def get_rez_version(self, venv_session=None):
"""Returns rez version installed in this revision
Returns:
str or None if not found.
"""
rez_location = self.locate_rez_lib(venv_session)
if rez_location is None:
return
version_py = rez_location / "rez" / "utils" / "_version.py"
if version_py.is_file():
_locals = {"_rez_version": ""}
with open(str(version_py)) as f:
exec(f.read(), globals(), _locals)
return _locals["_rez_version"]
@_require_local # noqa
def production_bin_dir(self, venv_name):
"""Returns production bin scripts dir in this revision
Returns:
pathlib.Path, but the directory may not exists.
"""
bin_dirname = "Scripts" if platform.system() == "Windows" else "bin"
venv_bin_dir = self.path() / "venv" / venv_name / bin_dirname
return venv_bin_dir / "rez"
@_require_local # noqa
def production_bin_dirs(self):
bin_dirs = []
metadata = self.metadata()
if metadata and not metadata.get("rezup_version"):
# rezup-1.x
_log.debug("Is a 'rezup-1.x' styled revision.")
bin_dirs.append(self._path / "bin")
else:
# rezup-2.x
_log.debug("Is a 'rezup-2.x' styled revision.")
for venv_name in metadata.get("venvs", []):
bin_dirs.append(self.production_bin_dir(venv_name))
return bin_dirs
get_rez_version(self, venv_session=None)
Returns rez version installed in this revision
Returns:
| Type | Description |
|---|---|
str or None if not found. |
Source code in rezup/container.py
@_require_local # noqa
def get_rez_version(self, venv_session=None):
"""Returns rez version installed in this revision
Returns:
str or None if not found.
"""
rez_location = self.locate_rez_lib(venv_session)
if rez_location is None:
return
version_py = rez_location / "rez" / "utils" / "_version.py"
if version_py.is_file():
_locals = {"_rez_version": ""}
with open(str(version_py)) as f:
exec(f.read(), globals(), _locals)
return _locals["_rez_version"]
locate_rez_lib(self, venv_session=None)
Returns rez module location in this revision
Returns:
| Type | Description |
|---|---|
pathlib.Path or None if not found. |
Source code in rezup/container.py
@_require_local # noqa
def locate_rez_lib(self, venv_session=None):
"""Returns rez module location in this revision
Returns:
pathlib.Path or None if not found.
"""
if venv_session is None:
venv_path = self.path() / "venv" / "rez"
venv_session = virtualenv.session_via_cli(args=[str(venv_path)])
venv_lib = venv_session.creator.purelib
# rez may get installed in edit mode, try short route first
egg_link = venv_lib / "rez.egg-link"
if egg_link.is_file():
with open(str(egg_link), "r") as f:
package_location = f.readline().strip()
if os.path.isdir(package_location):
return Path(package_location)
for importer, modname, pkg in pkgutil.walk_packages([str(venv_lib)]):
if pkg and modname == "rez":
loader = importer.find_module(modname)
try:
path = loader.path # SourceFileLoader
return Path(path).parent.parent
except AttributeError:
path = loader.filename # ImpLoader, py2
return Path(path).parent
production_bin_dir(self, venv_name)
Returns production bin scripts dir in this revision
Returns:
| Type | Description |
|---|---|
pathlib.Path, but the directory may not exists. |
Source code in rezup/container.py
@_require_local # noqa
def production_bin_dir(self, venv_name):
"""Returns production bin scripts dir in this revision
Returns:
pathlib.Path, but the directory may not exists.
"""
bin_dirname = "Scripts" if platform.system() == "Windows" else "bin"
venv_bin_dir = self.path() / "venv" / venv_name / bin_dirname
return venv_bin_dir / "rez"
pull(self, check_out=True, fallback=False)
Return corresponding local side revision
If the revision is from remote container, calling this method will find a timestamp matched local revision and create one if not found by default.
If the revision is from local container, return self.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
check_out |
bool |
When no matched local revision, create one if True or just return None at the end. Default is True. |
True |
Returns:
| Type | Description |
|---|---|
Revision or None |
Source code in rezup/container.py
def pull(self, check_out=True, fallback=False):
"""Return corresponding local side revision
If the revision is from remote container, calling this method will
find a timestamp matched local revision and create one if not found
by default.
If the revision is from local container, return `self`.
Args:
check_out (bool, optional): When no matched local revision,
create one if True or just return None at the end.
Default is True.
fallback (bool, optional)
Returns:
Revision or None
"""
if not self.is_remote():
return self
# get local
_con_name = self._container.name()
_con_recipe = self._container.recipe() # careful, this affect's root
local = Container(_con_name, recipe=_con_recipe, force_local=True)
rev = local.get_revision_by_time(self._timestamp,
fallback=fallback,
only_ready=True)
_allow_create = rev is None and check_out
_did_fallback = rev.timestamp() != self._timestamp if rev else False
if not fallback and _allow_create:
_log.info("Pulling from remote container: %s" % self._container)
rev = Revision(container=local, dirname=self._dirname)
rev._write(pulling=self)
if fallback and _did_fallback:
_log.warning(
"Local revision at exact time (%s) is not ready, "
"fallback to %s" % (self.time_str(), rev)
)
if rev is not None:
rev._is_pulled = True
return rev
spawn_shell(self, command=None)
Spawn a sub-shell
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
command |
list |
Shell script file with args or commands. If given, the sub-shell will not be interactive. |
None |
Returns:
| Type | Description |
|---|---|
subprocess.Popen |
Source code in rezup/container.py
def spawn_shell(self, command=None):
"""Spawn a sub-shell
Args:
command (list, optional): Shell script file with args or commands.
If given, the sub-shell will not be interactive.
Returns:
subprocess.Popen
Raises:
ContainerError
"""
if not self.is_valid():
raise ContainerError("Cannot use invalid revision.")
if not self.is_ready():
raise ContainerError("Revision is not ready to be used.")
if self.is_remote():
# use local
revision = self.pull()
if revision is None:
raise ContainerError("No revision pulled.")
if not revision.is_ready():
raise ContainerError("Revision is not ready to be used.")
return revision.spawn_shell(command=command)
else:
# Launch subprocess
environment = self._compose_env()
shell_name, shell_exec = self._get_shell()
if command:
# run command and exit
if command[0] == ".":
cmd = command
else:
exe = command[0]
exe = shell.which(exe, env=environment) or exe
cmd = [exe] + command[1:]
else:
# interactive shell
_con_name = self._container.name()
_con_from = "remote" if self._is_pulled else "local"
prompt = "rezup (%s/%s) " % (_con_name, _con_from)
prompt = shell.format_prompt_code(prompt, shell_name)
environment.update({
"REZUP_PROMPT": os.getenv("REZUP_PROMPT", prompt),
})
cmd = shell.get_launch_cmd(
shell_name,
shell_exec,
interactive=True,
)
popen = subprocess.Popen(cmd, env=environment)
return popen
use(self, command=None, wait=True)
Run a sub-shell
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
command |
list |
Shell script with args or commands. If given, the sub-shell will not be interactive. |
None |
wait |
bool |
Whether to wait |
True |
Returns:
| Type | Description |
|---|---|
int |
subprocess return code, will always return 0 if |
Source code in rezup/container.py
def use(self, command=None, wait=True):
"""Run a sub-shell
Args:
command (list, optional): Shell script with args or commands. If
given, the sub-shell will not be interactive.
wait (bool, optional): Whether to wait `command` finish or not,
default True.
Returns:
int: subprocess return code, will always return 0 if `command`
is given and `wait` is False.
"""
block = not command
popen = self.spawn_shell(command=command)
if block or wait:
stdout, stderr = popen.communicate()
return popen.returncode
else:
return 0