Upload package dependencies

README.md

@@ -798,6 +798,26 @@ You can upload the generated packages automatically to a conan-server using the
         CONAN_STABLE_BRANCH_PATTERN: "release/*"
 
 
+## Upload dependencies ([#237](https://github.com/conan-io/conan-package-tools/issues/237))
+
+Sometimes your dependencies are not available in remotes and you need to pass ``--build=missing`` to build them.
+The problem is that you will need to fix one-by-one, updating the CI script, instead of just uploading all built packages.
+
+Now you can upload **ALL** of your dependencies together, in addition to your package, to the same remote. To do this, you need to define:
+
+    CONAN_UPLOAD_DEPENDENCIES="all"
+
+Or, set it in ``ConanMultiPackager`` arguments:
+
+    ConanMultiPackager(upload_dependencies="all")
+
+However, maybe you want to upload **ONLY** specified packages by their names:
+
+    CONAN_UPLOAD_DEPENDENCIES="foo/0.1@user/channel,bar/1.2@bar/channel"
+
+Or,
+
+    ConanMultiPackager(upload_dependencies=["foo/0.1@user/channel", "bar/1.2@bar/channel"])
 
 ## Pagination
 
@@ -1022,6 +1042,7 @@ Using **CONAN_CLANG_VERSIONS** env variable in Travis ci or Appveyor:
 - **upload_retry**: Num retries in upload in case of failure.
 - **upload_only_when_stable**: Will try to upload only if the channel is the stable channel. Default [False]
 - **upload_only_when_tag**: Will try to upload only if the branch is a tag. Default [False]
+- **upload_dependencies**: Will try to upload dependencies to your remote. Default [False]
 - **build_types**: List containing specific build types. Default ["Release", "Debug"]
 - **skip_check_credentials**: Conan will skip checking the user credentials before building the packages. And if no user/remote is specified, will try to upload with the
   already stored credentiales in the local cache. Default [False]
@@ -1122,6 +1143,7 @@ This is especially useful for CI integration.
 - **CONAN_UPLOAD_RETRY**: If defined, in case of fail retries to upload again the specified times
 - **CONAN_UPLOAD_ONLY_WHEN_STABLE**: If defined, will try to upload the packages only when the current channel is the stable one.
 - **CONAN_UPLOAD_ONLY_WHEN_TAG**: If defined, will try to upload the packages only when the current branch is a tag.
+- **CONAN_UPLOAD_DEPENDENCIES**: If defined, will try to upload the listed package dependencies to your remote.
 
 - **CONAN_SKIP_CHECK_CREDENTIALS**: Conan will skip checking the user credentials before building the packages. And if no user/remote is specified, will try to upload with the
   already stored credentiales in the local cache. Default [False]

cpt/packager.py

@@ -102,7 +102,8 @@ def __init__(self, username=None, channel=None, runner=None,
                  out=None,
                  test_folder=None,
                  cwd=None,
-                 config_url=None):
+                 config_url=None,
+                 upload_dependencies=None):
 
         self.printer = Printer(out)
         self.printer.print_rule()
@@ -252,6 +253,12 @@ def __init__(self, username=None, channel=None, runner=None,
 
         self.pip_install = pip_install or split_colon_env("CONAN_PIP_INSTALL")
 
+        self.upload_dependencies = upload_dependencies or split_colon_env("CONAN_UPLOAD_DEPENDENCIES") or ""
+        if isinstance(self.upload_dependencies, list):
+            self.upload_dependencies = ",".join(self.upload_dependencies)
+        if "all" in self.upload_dependencies and self.upload_dependencies != "all":
+            raise Exception("Upload dependencies only accepts or 'all' or package references. Do not mix both!")
+
         os.environ["CONAN_CHANNEL"] = self.channel
 
         if docker_32_images is not None:
@@ -525,7 +532,8 @@ def run_builds(self, curpage=None, total_pages=None, base_profile_name=None):
                                  printer=self.printer,
                                  upload=self._upload_enabled(),
                                  test_folder=self.test_folder,
-                                 config_url=self.config_url)
+                                 config_url=self.config_url,
+                                 upload_dependencies=self.upload_dependencies)
                 r.run()
             else:
                 docker_image = self._get_docker_image(build)
@@ -548,7 +556,8 @@ def run_builds(self, curpage=None, total_pages=None, base_profile_name=None):
                                        lcow_user_workaround=self.lcow_user_workaround,
                                        test_folder=self.test_folder,
                                        pip_install=self.pip_install,
-                                       config_url=self.config_url)
+                                       config_url=self.config_url,
+                                       upload_dependencies=self.upload_dependencies)
 
                 r.run(pull_image=not pulled_docker_images[docker_image],
                       docker_entry_script=self.docker_entry_script)

cpt/run_in_docker.py

@@ -33,6 +33,7 @@ def run():
     abs_profile_path = save_profile_to_tmp(profile_text)
     base_profile_text = unscape_env(os.getenv("CPT_BASE_PROFILE"))
     config_url = unscape_env(os.getenv("CPT_CONFIG_URL"))
+    upload_dependencies = unscape_env(os.getenv("CPT_UPLOAD_DEPENDENCIES"))
     if base_profile_text:
         base_profile_name = unscape_env(os.getenv("CPT_BASE_PROFILE_NAME"))
         tools.save(os.path.join(client_cache.profiles_path, base_profile_name),
@@ -41,7 +42,8 @@ def run():
     upload = os.getenv("CPT_UPLOAD_ENABLED")
     runner = CreateRunner(abs_profile_path, reference, conan_api, uploader,
                           build_policy=build_policy, printer=printer, upload=upload,
-                          test_folder=test_folder, config_url=config_url)
+                          test_folder=test_folder, config_url=config_url,
+                          upload_dependencies=upload_dependencies)
     runner.run()
 
 

cpt/runner.py

@@ -15,7 +15,8 @@ class CreateRunner(object):
 
     def __init__(self, profile_abs_path, reference, conan_api, uploader,
                  exclude_vcvars_precommand=False, build_policy=None, runner=None,
-                 cwd=None, printer=None, upload=False, test_folder=None, config_url=None):
+                 cwd=None, printer=None, upload=False, test_folder=None, config_url=None,
+                 upload_dependencies=None):
 
         self.printer = printer or Printer()
         self._cwd = cwd or os.getcwd()
@@ -30,6 +31,9 @@ def __init__(self, profile_abs_path, reference, conan_api, uploader,
         self._uploader.remote_manager.add_remotes_to_conan()
         self._test_folder = test_folder
         self._config_url = config_url
+        self._upload_dependencies = upload_dependencies.split(",") if \
+                                    isinstance(upload_dependencies, str) else \
+                                    upload_dependencies
 
         patch_default_base_profile(conan_api, profile_abs_path)
 
@@ -104,10 +108,14 @@ def run(self):
                             self.printer.print_rule()
                             return
                         for installed in r['installed']:
-                            if installed["recipe"]["id"] == str(self._reference):
+                            reference = installed["recipe"]["id"]
+                            if ((reference == str(self._reference)) or \
+                               (reference in self._upload_dependencies) or \
+                               ("all" in self._upload_dependencies)) and \
+                               installed['packages']:
                                 package_id = installed['packages'][0]['id']
                                 if installed['packages'][0]["built"]:
-                                    self._uploader.upload_packages(self._reference,
+                                    self._uploader.upload_packages(reference,
                                                                    self._upload, package_id)
                                 else:
                                     self.printer.print_message("Skipping upload for %s, "
@@ -127,7 +135,8 @@ def __init__(self, profile_text, base_profile_text, base_profile_name, reference
                  docker_platform_param="", lcow_user_workaround="",
                  test_folder=None,
                  pip_install=None,
-                 config_url=None):
+                 config_url=None,
+                 upload_dependencies=None):
 
         self.printer = Printer()
         self._upload = upload
@@ -152,6 +161,7 @@ def __init__(self, profile_text, base_profile_text, base_profile_name, reference
         self._test_folder = test_folder
         self._pip_install = pip_install
         self._config_url = config_url
+        self._upload_dependencies = upload_dependencies
 
     def _pip_update_conan_command(self):
         commands = []
@@ -259,6 +269,7 @@ def get_env_vars(self):
         ret["CPT_BUILD_POLICY"] = escape_env(self._build_policy)
         ret["CPT_TEST_FOLDER"] = escape_env(self._test_folder)
         ret["CPT_CONFIG_URL"] = escape_env(self._config_url)
+        ret["CPT_UPLOAD_DEPENDENCIES"] = escape_env(self._upload_dependencies)
 
         ret.update({key: value for key, value in os.environ.items() if key.startswith("PIP_")})
 

cpt/test/test_client/upload_checks_test.py

@@ -101,3 +101,133 @@ def test_upload_when_tag_is_true(self):
             self.assertIn("Redefined channel by branch tag", tc.out)
             self.assertIn("Uploading packages for 'lib/1.0@user/stable'", tc.out)
             self.assertIn("Uploading package 1/1: 5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9 to 'default'", tc.out)
+
+
+class UploadDependenciesTest(unittest.TestCase):
+
+    conanfile_bar = """from conans import ConanFile
+class Pkg(ConanFile):
+    name = "bar"
+    version = "0.1.0"
+
+    def build(self):
+        pass
+    """
+
+    conanfile_foo = """from conans import ConanFile
+class Pkg(ConanFile):
+    name = "foo"
+    version = "1.0.0"
+    options = {"shared": [True, False]}
+    default_options = "shared=True"
+
+    def build(self):
+        pass
+    """
+
+    conanfile = """from conans import ConanFile
+class Pkg(ConanFile):
+    name = "foobar"
+    version = "2.0"
+    requires = "bar/0.1.0@foo/stable", "foo/1.0.0@bar/testing"
+
+    def build(self):
+        self.output.warn("BUILDING")
+"""
+
+    def setUp(self):
+        self._server = TestServer(users={"user": "password"},
+                                  write_permissions=[("bar/0.1.0@foo/stable", "user"),
+                                                     ("foo/1.0.0@bar/testing", "user")])
+        self._client = TestClient(servers={"default": self._server},
+                                 users={"default": [("user", "password")]})
+        self._client.save({"conanfile_bar.py": self.conanfile_bar})
+        self._client.run("export conanfile_bar.py foo/stable")
+        self._client.save({"conanfile_foo.py": self.conanfile_foo})
+        self._client.run("export conanfile_foo.py bar/testing")
+        self._client.save({"conanfile.py": self.conanfile})
+
+    def test_upload_all_dependencies(self):
+        with environment_append({"CONAN_UPLOAD":  self._server.fake_url,
+                                 "CONAN_LOGIN_USERNAME": "user",
+                                 "CONAN_PASSWORD": "password", "CONAN_USERNAME": "user",
+                                 "CONAN_UPLOAD_DEPENDENCIES": "all"}):
+
+            mulitpackager = get_patched_multipackager(self._client, username="user",
+                                                      channel="testing",
+                                                      build_policy="missing",
+                                                      exclude_vcvars_precommand=True)
+            mulitpackager.add({}, {})
+            mulitpackager.run()
+
+            self.assertIn("Uploading packages for 'foobar/2.0@user/testing'", self._client.out)
+            self.assertIn("Uploaded conan recipe 'foobar/2.0@user/testing'", self._client.out)
+            self.assertIn("Uploading package 1/1: f88b82969cca9c4bf43f9effe1157e641f38f16d", self._client.out)
+
+            self.assertIn("Uploading packages for 'bar/0.1.0@foo/stable'", self._client.out)
+            self.assertIn("Uploaded conan recipe 'bar/0.1.0@foo/stable'", self._client.out)
+            self.assertIn("Uploading package 1/1: 5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", self._client.out)
+
+            self.assertIn("Uploading packages for 'foo/1.0.0@bar/testing'", self._client.out)
+            self.assertIn("Uploaded conan recipe 'foo/1.0.0@bar/testing'", self._client.out)
+            self.assertIn("Uploading package 1/1: 2a623e3082a38f90cd2c3d12081161412de331b0", self._client.out)
+
+    def test_invalid_upload_dependencies(self):
+        with environment_append({"CONAN_UPLOAD":  self._server.fake_url,
+                                 "CONAN_LOGIN_USERNAME": "user",
+                                 "CONAN_PASSWORD": "password", "CONAN_USERNAME": "user",
+                                 "CONAN_UPLOAD_DEPENDENCIES": "all,bar/0.1.0@foo/stable"}):
+            with self.assertRaises(Exception) as context:
+                get_patched_multipackager(self._client, exclude_vcvars_precommand=True)
+            self.assertIn("Upload dependencies only accepts or 'all' or package references. Do not mix both!", str(context.exception))
+
+
+    def test_upload_specific_dependencies(self):
+        with environment_append({"CONAN_UPLOAD":  self._server.fake_url,
+                                 "CONAN_LOGIN_USERNAME": "user",
+                                 "CONAN_PASSWORD": "password", "CONAN_USERNAME": "user",
+                                 "CONAN_UPLOAD_DEPENDENCIES": "foo/1.0.0@bar/testing"}):
+
+            mulitpackager = get_patched_multipackager(self._client, username="user",
+                                                      channel="testing",
+                                                      build_policy="missing",
+                                                      exclude_vcvars_precommand=True)
+            mulitpackager.add({}, {})
+            mulitpackager.run()
+
+            self.assertIn("Uploading packages for 'foobar/2.0@user/testing'", self._client.out)
+            self.assertIn("Uploaded conan recipe 'foobar/2.0@user/testing'", self._client.out)
+            self.assertIn("Uploading package 1/1: f88b82969cca9c4bf43f9effe1157e641f38f16d", self._client.out)
+
+            self.assertNotIn("Uploading packages for 'bar/0.1.0@foo/stable'", self._client.out)
+            self.assertNotIn("Uploaded conan recipe 'bar/0.1.0@foo/stable'", self._client.out)
+            self.assertNotIn("Uploading package 1/1: 5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", self._client.out)
+
+            self.assertIn("Uploading packages for 'foo/1.0.0@bar/testing'", self._client.out)
+            self.assertIn("Uploaded conan recipe 'foo/1.0.0@bar/testing'", self._client.out)
+            self.assertIn("Uploading package 1/1: 2a623e3082a38f90cd2c3d12081161412de331b0", self._client.out)
+
+    def test_upload_regex_dependencies(self):
+        with environment_append({"CONAN_UPLOAD":  self._server.fake_url,
+                                 "CONAN_LOGIN_USERNAME": "user",
+                                 "CONAN_PASSWORD": "password", "CONAN_USERNAME": "user",
+                                 "CONAN_UPLOAD_DEPENDENCIES": "foo/*"}):
+
+            mulitpackager = get_patched_multipackager(self._client, username="user",
+                                                      channel="testing",
+                                                      build_policy="missing",
+                                                      exclude_vcvars_precommand=True)
+            mulitpackager.add({}, {})
+            mulitpackager.run()
+
+            self.assertIn("Uploading packages for 'foobar/2.0@user/testing'", self._client.out)
+            self.assertIn("Uploaded conan recipe 'foobar/2.0@user/testing'", self._client.out)
+            self.assertIn("Uploading package 1/1: f88b82969cca9c4bf43f9effe1157e641f38f16d", self._client.out)
+
+            self.assertNotIn("Uploading packages for 'bar/0.1.0@foo/stable'", self._client.out)
+            self.assertNotIn("Uploaded conan recipe 'bar/0.1.0@foo/stable'", self._client.out)
+            self.assertNotIn("Uploading package 1/1: 5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", self._client.out)
+
+            self.assertNotIn("Uploading packages for 'foo/1.0.0@bar/testing'", self._client.out)
+            self.assertNotIn("Uploaded conan recipe 'foo/1.0.0@bar/testing'", self._client.out)
+            self.assertNotIn("Uploading package 1/1: 2a623e3082a38f90cd2c3d12081161412de331b0", self._client.out)