Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs/deploying.rst
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,11 @@ files, and new untracked files that aren't gitignored, are still included.
Projects that aren't inside a git repository deploy the working directory
as-is, unchanged from before.

To deploy only the files committed to git ``HEAD``, leaving out uncommitted
changes and untracked files, use ``--from-head``::

shub deploy --from-head


.. _deploying-dependencies:

Expand Down
55 changes: 48 additions & 7 deletions shub/build_egg.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
import io
import os
import shutil
import subprocess
import tarfile
import tempfile
from pathlib import Path

Expand Down Expand Up @@ -35,7 +38,9 @@
the working directory that leaves out anything git considers ignored (e.g.
via .gitignore), so build artifacts, local secrets, and stray virtualenvs
don't end up in the egg. Uncommitted changes to tracked files are still
included.
included. To build the egg only from what is committed, use:

shub build-egg --from-head myproject.egg
"""

SHORT_HELP = "Build the project egg without deploying it"
Expand All @@ -48,15 +53,20 @@
@click.option(
"-d", "--debug", help="Debug mode (do not remove build dir)", is_flag=True
)
def cli(filename, debug):
build_egg_cmd(filename, debug)
@click.option(
"--from-head",
is_flag=True,
help="Build the egg from the files committed to git HEAD",
)
def cli(filename, debug, from_head):
build_egg_cmd(filename, debug, from_head)


def build_egg_cmd(filename, debug=False):
def build_egg_cmd(filename, debug=False, from_head=False):
tmpdir = None
try:
with _project_cwd():
egg, tmpdir = build_project_egg()
egg, tmpdir = build_project_egg(from_head)
click.echo(f"Writing egg to {filename}")
try:
shutil.copyfile(egg, filename)
Expand All @@ -68,9 +78,10 @@ def build_egg_cmd(filename, debug=False):
remove_build_dir(tmpdir, debug)


def build_project_egg():
def build_project_egg(from_head=False):
"""
Build the egg of the Scrapy project that contains the current directory.
Build the egg of the Scrapy project that contains the current directory,
from the files committed to git HEAD if *from_head* is true.

Return the path to the egg and the temporary directory it was built in.
Callers are expected to clean up that directory afterwards, see
Expand All @@ -79,6 +90,10 @@ def build_project_egg():
if not inside_project():
raise NotFoundException("No Scrapy project found in this location.")
git = shutil.which("git")
if from_head:
if not git:
raise NotFoundException("--from-head requires git.")
return _build_egg_from_git_head(git)
if git:
try:
return _build_egg_from_git_filtered_files(git)
Expand Down Expand Up @@ -141,3 +156,29 @@ def _build_egg_from_git_filtered_files(git):
return _build_egg_in_cwd()
finally:
shutil.rmtree(filtered_dir, ignore_errors=True)


def _build_egg_from_git_head(git):
"""Build the egg from the files under the current directory as committed
to git HEAD."""
try:
archive = subprocess.run( # noqa: S603
[git, "archive", "HEAD"],
capture_output=True,
check=True,
).stdout
except subprocess.CalledProcessError as e:
raise SubcommandException(
"Error while calling 'git archive HEAD':\n\n"
+ e.stderr.decode(errors="replace")
) from e
head_dir = tempfile.mkdtemp(prefix="shub-deploy-head-")
try:
with tarfile.open(fileobj=io.BytesIO(archive)) as tar:
tar.extraction_filter = getattr(tarfile, "data_filter", None)
tar.extractall(head_dir) # noqa: S202
with remember_cwd():
os.chdir(head_dir)
return _build_egg_in_cwd()
finally:
shutil.rmtree(head_dir, ignore_errors=True)
36 changes: 28 additions & 8 deletions shub/deploy.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,9 @@
the working directory that leaves out anything git considers ignored (e.g.
via .gitignore), so build artifacts, local secrets, and stray virtualenvs
don't end up in the deploy. Uncommitted changes to tracked files are still
included.
included. To deploy only what is committed, use:

shub deploy --from-head
"""

SHORT_HELP = "Deploy Scrapy project to Scrapy Cloud"
Expand Down Expand Up @@ -103,15 +105,31 @@ def _deprecate_build_egg_parameter(ctx, param, value):
@click.option(
"--ignore-size", help="Ignore deploy request's egg(s) size check", is_flag=True
)
def cli(target, version, debug, egg, build_egg, verbose, keep_log, ignore_size):
@click.option(
"--from-head",
is_flag=True,
help="Build the egg from the files committed to git HEAD",
)
def cli(
target, version, debug, egg, build_egg, verbose, keep_log, ignore_size, from_head
):
if build_egg is not None:
build_egg_cmd(build_egg, debug)
build_egg_cmd(build_egg, debug, from_head)
return
conf = load_shub_config()
create_scrapinghub_yml_wizard(conf, target=target)
image = conf.get_target_conf(target).image
if not image:
deploy_cmd(target, version, debug, egg, verbose, keep_log, conf=conf)
deploy_cmd(
target,
version,
debug,
egg,
verbose,
keep_log,
conf=conf,
from_head=from_head,
)
elif image.startswith(SH_IMAGES_REGISTRY):
upload_cmd(target, version)
else:
Expand All @@ -121,13 +139,15 @@ def cli(target, version, debug, egg, build_egg, verbose, keep_log, ignore_size):
)


def deploy_cmd(target, version, debug, egg, verbose, keep_log, conf=None):
def deploy_cmd(
target, version, debug, egg, verbose, keep_log, conf=None, from_head=False
):
egg = egg and str(Path(egg).absolute())
with _project_cwd():
_deploy_cmd(target, version, debug, egg, verbose, keep_log, conf)
_deploy_cmd(target, version, debug, egg, verbose, keep_log, conf, from_head)


def _deploy_cmd(target, version, debug, egg, verbose, keep_log, conf):
def _deploy_cmd(target, version, debug, egg, verbose, keep_log, conf, from_head):
tmpdir = None
try:
conf = conf or load_shub_config()
Expand All @@ -139,7 +159,7 @@ def _deploy_cmd(target, version, debug, egg, verbose, keep_log, conf):
click.echo(f"Using egg: {egg}")
else:
click.echo(f"Packing version {version}")
egg, tmpdir = build_project_egg()
egg, tmpdir = build_project_egg(from_head)

_upload_egg(
targetconf.endpoint,
Expand Down
50 changes: 49 additions & 1 deletion tests/test_build_egg.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,11 @@
from pathlib import Path
from unittest import mock

import pytest
from click.testing import CliRunner

from shub import build_egg
from shub.exceptions import NotFoundException, ShubException
from shub.exceptions import NotFoundException, ShubException, SubcommandException

from .utils import VALID_SCRAPY_CFG, AssertInvokeRaisesMixin, make_git_project

Expand Down Expand Up @@ -80,6 +81,39 @@ def test_build_egg_falls_back_without_git_repo(self):
shutil.rmtree(tmpdir, ignore_errors=True)
assert egg.endswith(".egg")

def test_build_egg_from_head(self):
with self.runner.isolated_filesystem():
make_git_project()
with Path("project", "__init__.py").open("w") as f:
f.write("MARKER = 1\n")
with Path("project", "extra.py").open("w") as f:
f.write("EXTRA = 1\n")
egg, tmpdir = build_egg.build_project_egg(from_head=True)
try:
with zipfile.ZipFile(egg) as z:
names = z.namelist()
member = next(n for n in names if n.endswith("__init__.py"))
content = z.read(member)
finally:
shutil.rmtree(tmpdir, ignore_errors=True)
assert b"MARKER" not in content
assert not any("extra" in n for n in names)

def test_build_egg_from_head_without_git_repo(self):
with self.runner.isolated_filesystem():
with Path("scrapy.cfg").open("w") as f:
f.write(VALID_SCRAPY_CFG)
with pytest.raises(SubcommandException):
build_egg.build_project_egg(from_head=True)

@mock.patch("shub.build_egg.shutil.which", return_value=None)
def test_build_egg_from_head_without_git(self, mock_which):
with self.runner.isolated_filesystem():
with Path("scrapy.cfg").open("w") as f:
f.write(VALID_SCRAPY_CFG)
with pytest.raises(NotFoundException):
build_egg.build_project_egg(from_head=True)


class RemoveBuildDirTest(unittest.TestCase):
def setUp(self):
Expand Down Expand Up @@ -174,6 +208,20 @@ def test_neither_uploads_nor_needs_config(
mock_deploy_conf.assert_not_called()
mock_conf.assert_not_called()

def test_from_head(self):
if shutil.which("git") is None:
self.skipTest("git executable not found")
with self.runner.isolated_filesystem():
make_git_project()
with Path("project", "extra.py").open("w") as f:
f.write("EXTRA = 1\n")
result = self.runner.invoke(build_egg.cli, ["--from-head", "built.egg"])
assert 0 == result.exit_code, result.output
with zipfile.ZipFile("built.egg") as z:
names = z.namelist()
assert "project/__init__.py" in names
assert not any("extra" in n for n in names)

def test_removes_build_dir(self):
patcher, tmpdir = self._fake_egg_build()
with patcher, self.runner.isolated_filesystem():
Expand Down
9 changes: 9 additions & 0 deletions tests/test_deploy.py
Original file line number Diff line number Diff line change
Expand Up @@ -338,6 +338,15 @@ def test_deploy_default_from_git_repo(self, mock_deploy_req):
result = self.runner.invoke(deploy.cli)
assert 0 == result.exit_code, result.output

@patch("shub.deploy.build_project_egg", wraps=build_project_egg)
@patch("shub.deploy.make_deploy_request")
def test_deploy_from_head_flag(self, mock_deploy_req, mock_build):
with self.runner.isolated_filesystem():
make_git_project()
result = self.runner.invoke(deploy.cli, ["--from-head"])
assert 0 == result.exit_code, result.output
mock_build.assert_called_once_with(True)

@patch("shub.deploy.make_deploy_request")
def test_deploy_from_subdir(self, mock_deploy_req):
with self.runner.isolated_filesystem():
Expand Down
Loading