Merge branch 'use-pulp-v3' into 'master'

Use pulp v3

See merge request MESA_Platform/build-env!1

6 files changed, 242 insertions, 22 deletions

diff --git a/Dockerfile b/Dockerfile
index 55293cf..7b4f817 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,20 +1,14 @@
 FROM centos:7
+COPY rpm_upload_tools.py /root/rpm_upload_tools.py
+COPY repo.internal.geedge.net.repo /etc/yum.repos.d/repo.internal.geedge.net.repo
+COPY netrc.conf /root/.netrc
 
-RUN yum install -y gcc gcc-c++ make libpcap-devel epel-release patch wget socat automake autoconf rpm-build git && \
-    wget https://repos.fedorapeople.org/repos/pulp/pulp/rhel-pulp.repo -O /etc/yum.repos.d/rhel-pulp.repo && \
-    yum install -y pulp-consumer-client pulp-rpm-consumer-extensions pulp-puppet-consumer-extensions pulp-agent && \
-    yum install -y pulp-rpm-handlers pulp-rpm-yumplugins pulp-puppet-handlers python-gofer-qpid && \
-    yum install -y pulp-admin-client pulp-rpm-admin-extensions pulp-puppet-admin-extensions pulp-docker-admin-extensions && \
-    yum install -y python-pip && \
-    pip install -i https://mirrors.aliyun.com/pypi/simple/ --no-cache-dir cmake && \
+RUN yum install -y gcc gcc-c++ make libpcap-devel epel-release patch wget socat automake autoconf libtool rpm-build git && \
+    yum install -y cmake3 && \
+    yum install -y python3-pip && \
+    pip3 install -i https://pypi.tuna.tsinghua.edu.cn/simple httpie && \
     curl -sL https://sentry.io/get-cli/ | bash && \
     yum clean all && \
     rm -rf /var/cache/yum
 
-ADD repo.mesalab.cn.conf /etc/pulp/consumer/conf.d/repo.mesalab.cn.conf
-ADD repo.mesalab.cn.conf /etc/pulp/admin/conf.d/repo.mesalab.cn.conf
-ADD docker-entrypoint.sh /docker-entrypoint.sh
-RUN chmod +x /docker-entrypoint.sh
-
-ENTRYPOINT ["/docker-entrypoint.sh"]
-CMD ["/bin/bash"]
\ No newline at end of file
+CMD [ "/bin/bash" ]
diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh
deleted file mode 100755
index 669f976..0000000
--- a/docker-entrypoint.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/bash
-set -e
-socat UNIX-RECV:/dev/log,mode=666 STDOUT &
-/usr/bin/goferd
-exec "$@"
diff --git a/netrc.conf b/netrc.conf
new file mode 100644
index 0000000..20815da
--- /dev/null
+++ b/netrc.conf
@@ -0,0 +1,3 @@
+machine repo.internal.geedge.net
+login admin
+password password
\ No newline at end of file
diff --git a/repo.internal.geedge.net.repo b/repo.internal.geedge.net.repo
new file mode 100644
index 0000000..f2e1b58
--- /dev/null
+++ b/repo.internal.geedge.net.repo
@@ -0,0 +1,4 @@
+[mesa-platform]
+name=mesa-platform
+baseurl=http://repo.internal.geedge.net/pulp/content/mesa-platform-stable/
+gpgcheck=0
diff --git a/repo.mesalab.cn.conf b/repo.mesalab.cn.conf
deleted file mode 100644
index 7c6c070..0000000
--- a/repo.mesalab.cn.conf
+++ /dev/null
@@ -1,3 +0,0 @@
-[server]
-host: repo.mesalab.cn
-port: 443
diff --git a/rpm_upload_tools.py b/rpm_upload_tools.py
new file mode 100644
index 0000000..49a9b2b
--- /dev/null
+++ b/rpm_upload_tools.py
@@ -0,0 +1,227 @@
+import requests
+import urllib
+import json
+import time
+import logging
+import argparse
+import os
+
+PULP_SERVER_URL_BASE = "http://repo.internal.geedge.net/"
+PULP_REPO_HREF = ""
+PULP_DIST_HREF = ""
+
+
+def orphans_cleanup():
+    api_cleanup = urllib.parse.urljoin(
+        PULP_SERVER_URL_BASE, '/pulp/api/v3/orphans/')
+
+    r = requests.delete(api_cleanup)
+    r.raise_for_status()
+
+
+def wait_until_task_finished(task_url):
+    api_task_url = urllib.parse.urljoin(PULP_SERVER_URL_BASE, task_url)
+    while True:
+        r = requests.get(api_task_url)
+        r.raise_for_status()
+        state = r.json()['state']
+
+        if state == 'failed' or state == 'cancelled':
+            raise Exception("task is cancelled", task_url, state)
+        if state == 'completed':
+            break
+
+        # wait for task to complete
+        time.sleep(1)
+    return
+
+
+def task_created_resource_by_task_url(task_url):
+    r = requests.get(urllib.parse.urljoin(PULP_SERVER_URL_BASE, task_url))
+    r.raise_for_status()
+    return json.loads(r.text)['created_resources'][0]
+
+
+def artifact_create(path):
+    api_url_artifact = urllib.parse.urljoin(
+        PULP_SERVER_URL_BASE, "/pulp/api/v3/artifacts/")
+    with open(path, 'rb') as fp:
+        files = {'file': fp}
+        r = requests.post(api_url_artifact, files=files)
+        r.raise_for_status()
+
+    href = r.json()['pulp_href']
+    href_url = urllib.parse.urljoin(PULP_SERVER_URL_BASE, href)
+    requests.get(href_url).raise_for_status()
+    return href
+
+
+def content_create_from_artifact(artifact_href, pkgname):
+    api_url_create_content = urllib.parse.urljoin(
+        PULP_SERVER_URL_BASE, '/pulp/api/v3/content/rpm/packages/')
+    r = requests.post(api_url_create_content, data={
+                      'artifact': artifact_href, 'relative_path': pkgname})
+    r.raise_for_status()
+
+    task_href = r.json()['task']
+    task_full_url = urllib.parse.urljoin(PULP_SERVER_URL_BASE, task_href)
+    wait_until_task_finished(task_full_url)
+
+    r = requests.get(task_full_url)
+    r.raise_for_status()
+
+    package_href = json.loads(r.text)['created_resources'][0]
+    r = requests.get(urllib.parse.urljoin(PULP_SERVER_URL_BASE, package_href))
+    r.raise_for_status()
+    return package_href
+
+
+def add_content_to_repo(repo_href, package_hrefs):
+    api_url_publish = urllib.parse.urljoin(
+        PULP_SERVER_URL_BASE, repo_href + 'modify/')
+    r = requests.post(api_url_publish, json={
+                      'add_content_units': package_hrefs})
+    r.raise_for_status()
+
+    # create task and wait for it to complete
+    task_href = r.json()['task']
+    task_full_url = urllib.parse.urljoin(PULP_SERVER_URL_BASE, task_href)
+    wait_until_task_finished(task_full_url)
+
+
+def publication_repo(repo_href):
+    api_url_publish = urllib.parse.urljoin(
+        PULP_SERVER_URL_BASE, '/pulp/api/v3/publications/rpm/rpm/')
+
+    r = requests.post(api_url_publish, json={'repository': repo_href})
+    r.raise_for_status()
+
+    # wait the task
+    task_href = r.json()['task']
+    task_full_url = urllib.parse.urljoin(PULP_SERVER_URL_BASE, task_href)
+    wait_until_task_finished(task_full_url)
+
+    # publication
+    pub_href = task_created_resource_by_task_url(task_full_url)
+    return pub_href
+
+
+def distribution(pub_href, name, base_path):
+    api_url = urllib.parse.urljoin(
+        PULP_SERVER_URL_BASE, '/pulp/api/v3/distributions/rpm/rpm/')
+
+    r = requests.put(
+        api_url, json={'name': name, 'base_path': base_path, 'publication': pub_href})
+    r.raise_for_status()
+
+    # wait the task
+    task_href = r.json()['task']
+    task_full_url = urllib.parse.urljoin(PULP_SERVER_URL_BASE, task_href)
+    wait_until_task_finished(task_full_url)
+
+
+def update_distributation_pub(dist_href, pub_href):
+    api_url = urllib.parse.urljoin(PULP_SERVER_URL_BASE, dist_href)
+    r = requests.get(api_url)
+    r.raise_for_status()
+
+    dist_base_path = r.json()['base_path']
+    dist_name = r.json()['name']
+
+    logging.info('prepare to update dist, name: %s, base_path: %s' %
+                 (dist_name, dist_base_path))
+
+    r = requests.put(api_url, json={
+                     'base_path': dist_base_path, 'name': dist_name, 'publication': pub_href})
+    r.raise_for_status()
+
+    # wait the task
+    task_href = r.json()['task']
+    task_full_url = urllib.parse.urljoin(PULP_SERVER_URL_BASE, task_href)
+    wait_until_task_finished(task_full_url)
+
+    logging.info('update dist successfully, %s', task_href)
+
+
+def dist_href_lookup_by_name(distname):
+    api_url = urllib.parse.urljoin(
+        PULP_SERVER_URL_BASE, '/pulp/api/v3/distributions/rpm/rpm/')
+
+    r = requests.get(api_url)
+    r.raise_for_status()
+    logging.debug('response: %s', r.json())
+
+    for result in r.json()['results']:
+        if (result['name'].strip() == distname):
+            return result['pulp_href']
+
+    raise Exception('cannot lookup %s\'s dist href, Not exist.' % distname)
+
+
+def repo_href_lookup_by_name(reponame):
+    api_url = urllib.parse.urljoin(
+        PULP_SERVER_URL_BASE, '/pulp/api/v3/repositories/rpm/rpm/')
+
+    r = requests.get(api_url)
+    r.raise_for_status()
+
+    logging.fatal('response: %s', r.json())
+
+    for result in r.json()['results']:
+        if (result['name'] == reponame):
+            return result['pulp_href']
+
+    raise Exception('cannot lookup %s\'s repo href, Not exist.' % reponame)
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Pulp3 RPM upload helper')
+    parser.add_argument('repo', type=str)
+    parser.add_argument('dist', type=str)
+    parser.add_argument('package', type=str, nargs='+')
+    parser.add_argument('--pulp-server-url', type=str)
+    parser.add_argument('--verbose', default=False, action='store_true')
+    args = parser.parse_args()
+
+    if (args.verbose):
+        logging.basicConfig(level=logging.DEBUG)
+
+    dist_href = dist_href_lookup_by_name(args.dist)
+    repo_href = repo_href_lookup_by_name(args.repo)
+
+    logging.info('dist_href: %s' % dist_href)
+    logging.info('repo_href: %s' % repo_href)
+
+    if args.pulp_server_url:
+        global PULP_SERVER_URL_BASE
+        PULP_SERVER_URL_BASE = args.pulp_server_url
+
+    package_list = []
+    for package in args.package:
+        if os.path.isfile(package):
+            package_list.append(package)
+            continue
+
+        for file in os.listdir(package):
+            path = os.path.join(package, file)
+            if os.path.isfile(path) and path.endswith('.rpm'):
+                package_list.append(path)
+
+    logging.info('RPMS: %s', str(package_list))
+
+    orphans_cleanup()
+    package_href_collection = []
+    for package_path in package_list:
+        package_basename = os.path.basename(package_path)
+        artifact_href = artifact_create(package_path)
+        package_href = content_create_from_artifact(
+            artifact_href, package_basename)
+        package_href_collection.append(package_href)
+
+    add_content_to_repo(repo_href, package_href_collection)
+    pub_href = publication_repo(repo_href)
+    update_distributation_pub(dist_href, pub_href)
+
+
+if __name__ == "__main__":
+    main()