From ec1b08f1b36069a0308d712ea662aecddc8b32f8 Mon Sep 17 00:00:00 2001 From: Lu Qiuwen Date: Mon, 23 Dec 2019 17:39:02 +0800 Subject: 适配PULP3版本 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 17 ++++------------- docker-entrypoint.sh | 5 ----- repo.mesalab.cn.conf | 3 --- 3 files changed, 4 insertions(+), 21 deletions(-) delete mode 100755 docker-entrypoint.sh delete mode 100644 repo.mesalab.cn.conf diff --git a/Dockerfile b/Dockerfile index 55293cf..e6ada7f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,20 +1,11 @@ FROM centos:7 RUN yum install -y gcc gcc-c++ make libpcap-devel epel-release patch wget socat automake autoconf rpm-build git && \ - wget https://repos.fedorapeople.org/repos/pulp/pulp/rhel-pulp.repo -O /etc/yum.repos.d/rhel-pulp.repo && \ - yum install -y pulp-consumer-client pulp-rpm-consumer-extensions pulp-puppet-consumer-extensions pulp-agent && \ - yum install -y pulp-rpm-handlers pulp-rpm-yumplugins pulp-puppet-handlers python-gofer-qpid && \ - yum install -y pulp-admin-client pulp-rpm-admin-extensions pulp-puppet-admin-extensions pulp-docker-admin-extensions && \ - yum install -y python-pip && \ - pip install -i https://mirrors.aliyun.com/pypi/simple/ --no-cache-dir cmake && \ + yum install -y cmake3 && \ + yum install -y python3-pip && \ + pip3 install -i https://pypi.tuna.tsinghua.edu.cn/simple httpie && \ curl -sL https://sentry.io/get-cli/ | bash && \ yum clean all && \ rm -rf /var/cache/yum -ADD repo.mesalab.cn.conf /etc/pulp/consumer/conf.d/repo.mesalab.cn.conf -ADD repo.mesalab.cn.conf /etc/pulp/admin/conf.d/repo.mesalab.cn.conf -ADD docker-entrypoint.sh /docker-entrypoint.sh -RUN chmod +x /docker-entrypoint.sh - -ENTRYPOINT ["/docker-entrypoint.sh"] -CMD ["/bin/bash"] \ No newline at end of file +CMD [ "/bin/bash" ] \ No newline at end of file diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh deleted file mode 100755 index 669f976..0000000 --- a/docker-entrypoint.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -set -e -socat UNIX-RECV:/dev/log,mode=666 STDOUT & -/usr/bin/goferd -exec "$@" diff --git a/repo.mesalab.cn.conf b/repo.mesalab.cn.conf deleted file mode 100644 index 7c6c070..0000000 --- a/repo.mesalab.cn.conf +++ /dev/null @@ -1,3 +0,0 @@ -[server] -host: repo.mesalab.cn -port: 443 -- cgit v1.2.3 From 1afe4b76ade8642fa9387edeead56022f7c57d13 Mon Sep 17 00:00:00 2001 From: Lu Qiuwen Date: Wed, 25 Dec 2019 15:51:06 +0800 Subject: 集成RPM上传工具 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 3 +- rpm_upload_tools.py | 174 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 176 insertions(+), 1 deletion(-) create mode 100644 rpm_upload_tools.py diff --git a/Dockerfile b/Dockerfile index e6ada7f..30a7929 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM centos:7 -RUN yum install -y gcc gcc-c++ make libpcap-devel epel-release patch wget socat automake autoconf rpm-build git && \ +RUN yum install -y gcc gcc-c++ make libpcap-devel epel-release patch wget socat automake autoconf libtool rpm-build git && \ yum install -y cmake3 && \ yum install -y python3-pip && \ pip3 install -i https://pypi.tuna.tsinghua.edu.cn/simple httpie && \ @@ -8,4 +8,5 @@ RUN yum install -y gcc gcc-c++ make libpcap-devel epel-release patch wget socat yum clean all && \ rm -rf /var/cache/yum +COPY rpm_upload_tools.py /root/rpm_upload_tools.py CMD [ "/bin/bash" ] \ No newline at end of file diff --git a/rpm_upload_tools.py b/rpm_upload_tools.py new file mode 100644 index 0000000..f4c977e --- /dev/null +++ b/rpm_upload_tools.py @@ -0,0 +1,174 @@ +import requests +import urllib +import json +import time +import logging + +PULP_SERVER_URL_BASE = "http://repo.internal.geedge.net/" +PULP_REPO_HREF = "" +PULP_DIST_HREF = "" + + +def orphans_cleanup(): + api_cleanup = urllib.parse.urljoin( + PULP_SERVER_URL_BASE, '/pulp/api/v3/orphans/') + + r = requests.delete(api_cleanup) + r.raise_for_status() + + +def wait_until_task_finished(task_url): + api_task_url = urllib.parse.urljoin(PULP_SERVER_URL_BASE, task_url) + while True: + r = requests.get(api_task_url) + r.raise_for_status() + state = r.json()['state'] + + if state == 'failed' or state == 'cancelled': + raise Exception("task is cancelled", task_url, state) + if state == 'completed': + break + + # wait for task to complete + time.sleep(1) + return + + +def task_created_resource_by_task_url(task_url): + r = requests.get(urllib.parse.urljoin(PULP_SERVER_URL_BASE, task_url)) + r.raise_for_status() + return json.loads(r.text)['created_resources'][0] + + +def artifact_create(path): + api_url_artifact = urllib.parse.urljoin( + PULP_SERVER_URL_BASE, "/pulp/api/v3/artifacts/") + with open(path, 'rb') as fp: + files = {'file': fp} + r = requests.post(api_url_artifact, files=files) + r.raise_for_status() + + href = r.json()['pulp_href'] + href_url = urllib.parse.urljoin(PULP_SERVER_URL_BASE, href) + requests.get(href_url).raise_for_status() + return href + + +def content_create_from_artifact(artifact_href, pkgname): + api_url_create_content = urllib.parse.urljoin( + PULP_SERVER_URL_BASE, '/pulp/api/v3/content/rpm/packages/') + r = requests.post(api_url_create_content, data={ + 'artifact': artifact_href, 'relative_path': pkgname}) + r.raise_for_status() + + task_href = r.json()['task'] + task_full_url = urllib.parse.urljoin(PULP_SERVER_URL_BASE, task_href) + wait_until_task_finished(task_full_url) + + r = requests.get(task_full_url) + r.raise_for_status() + + package_href = json.loads(r.text)['created_resources'][0] + r = requests.get(urllib.parse.urljoin(PULP_SERVER_URL_BASE, package_href)) + r.raise_for_status() + return package_href + + +def add_content_to_repo(repo_href, package_hrefs): + api_url_publish = urllib.parse.urljoin( + PULP_SERVER_URL_BASE, repo_href + 'modify/') + r = requests.post(api_url_publish, json={ + 'add_content_units': package_hrefs}) + r.raise_for_status() + + # create task and wait for it to complete + task_href = r.json()['task'] + task_full_url = urllib.parse.urljoin(PULP_SERVER_URL_BASE, task_href) + wait_until_task_finished(task_full_url) + + +def publication_repo(repo_href): + api_url_publish = urllib.parse.urljoin( + PULP_SERVER_URL_BASE, '/pulp/api/v3/publications/rpm/rpm/') + + r = requests.post(api_url_publish, json={'repository': repo_href}) + r.raise_for_status() + + # wait the task + task_href = r.json()['task'] + task_full_url = urllib.parse.urljoin(PULP_SERVER_URL_BASE, task_href) + wait_until_task_finished(task_full_url) + + # publication + pub_href = task_created_resource_by_task_url(task_full_url) + return pub_href + + +def distribution(pub_href, name, base_path): + api_url = urllib.parse.urljoin( + PULP_SERVER_URL_BASE, '/pulp/api/v3/distributions/rpm/rpm/') + + r = requests.put( + api_url, json={'name': name, 'base_path': base_path, 'publication': pub_href}) + r.raise_for_status() + + # wait the task + task_href = r.json()['task'] + task_full_url = urllib.parse.urljoin(PULP_SERVER_URL_BASE, task_href) + wait_until_task_finished(task_full_url) + + +def update_distributation_pub(dist_href, pub_href): + api_url = urllib.parse.urljoin(PULP_SERVER_URL_BASE, dist_href) + r = requests.get(api_url) + r.raise_for_status() + + dist_base_path = r.json()['base_path'] + dist_name = r.json()['name'] + + logging.info('prepare to update dist, name: %s, base_path: %s' % + (dist_name, dist_base_path)) + + r = requests.put(api_url, json={ + 'base_path': dist_base_path, 'name': dist_name, 'publication': pub_href}) + r.raise_for_status() + + # wait the task + task_href = r.json()['task'] + task_full_url = urllib.parse.urljoin(PULP_SERVER_URL_BASE, task_href) + wait_until_task_finished(task_full_url) + + logging.info('update dist successfully, %s', task_href) + + +def main(): + import argparse + import os + + parser = argparse.ArgumentParser(description='Pulp3 RPM upload helper') + parser.add_argument('repo_href', type=str) + parser.add_argument('dist_href', type=str) + parser.add_argument('package', type=str, nargs='+') + parser.add_argument('--pulp-server-url', type=str) + args = parser.parse_args() + + if args.pulp_server_url: + global PULP_SERVER_URL_BASE + PULP_SERVER_URL_BASE = args.pulp_server_url + + orphans_cleanup() + package_href_collection = [] + for package_path in args.package: + package_basename = os.path.basename(package_path) + artifact_href = artifact_create(package_path) + package_href = content_create_from_artifact( + artifact_href, package_basename) + package_href_collection.append(package_href) + + add_content_to_repo(args.repo_href, package_href_collection) + pub_href = publication_repo(args.repo_href) + update_distributation_pub(args.dist_href, pub_href) + + +if __name__ == "__main__": + main() -- cgit v1.2.3 From 4428b44f6f68bdc03fc7d0ae1e2e761e08cc06d9 Mon Sep 17 00:00:00 2001 From: Lu Qiuwen Date: Wed, 25 Dec 2019 18:30:59 +0800 Subject: 改进上传脚本，由原来的传入HREF改为传入仓库名称 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rpm_upload_tools.py | 56 +++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 48 insertions(+), 8 deletions(-) diff --git a/rpm_upload_tools.py b/rpm_upload_tools.py index f4c977e..ddf1c35 100644 --- a/rpm_upload_tools.py +++ b/rpm_upload_tools.py @@ -3,6 +3,8 @@ import urllib import json import time import logging +import argparse +import os PULP_SERVER_URL_BASE = "http://repo.internal.geedge.net/" PULP_REPO_HREF = "" @@ -141,17 +143,55 @@ def update_distributation_pub(dist_href, pub_href): logging.info('update dist successfully, %s', task_href) -def main(): - import argparse - import os +def dist_href_lookup_by_name(distname): + api_url = urllib.parse.urljoin( + PULP_SERVER_URL_BASE, '/pulp/api/v3/distributions/rpm/rpm/') + + r = requests.get(api_url) + r.raise_for_status() + logging.debug('response: %s', r.json()) + + for result in r.json()['results']: + if (result['name'].strip() == distname): + return result['pulp_href'] + + raise Exception('cannot lookup %s\'s dist href, Not exist.' % distname) + + +def repo_href_lookup_by_name(reponame): + api_url = urllib.parse.urljoin( + PULP_SERVER_URL_BASE, '/pulp/api/v3/repositories/rpm/rpm/') + r = requests.get(api_url) + r.raise_for_status() + + logging.fatal('response: %s', r.json()) + + for result in r.json()['results']: + if (result['name'] == reponame): + return result['pulp_href'] + + raise Exception('cannot lookup %s\'s repo href, Not exist.' % reponame) + + +def main(): parser = argparse.ArgumentParser(description='Pulp3 RPM upload helper') - parser.add_argument('repo_href', type=str) - parser.add_argument('dist_href', type=str) + parser.add_argument('repo', type=str) + parser.add_argument('dist', type=str) parser.add_argument('package', type=str, nargs='+') parser.add_argument('--pulp-server-url', type=str) + parser.add_argument('--verbose', default=False, action='store_true') args = parser.parse_args() + if (args.verbose): + logging.basicConfig(level=logging.DEBUG) + + dist_href = dist_href_lookup_by_name(args.dist) + repo_href = repo_href_lookup_by_name(args.repo) + + logging.info('dist_href: %s' % dist_href) + logging.info('repo_href: %s' % repo_href) + if args.pulp_server_url: global PULP_SERVER_URL_BASE PULP_SERVER_URL_BASE = args.pulp_server_url @@ -165,9 +205,9 @@ def main(): artifact_href, package_basename) package_href_collection.append(package_href) - add_content_to_repo(args.repo_href, package_href_collection) - pub_href = publication_repo(args.repo_href) - update_distributation_pub(args.dist_href, pub_href) + add_content_to_repo(repo_href, package_href_collection) + pub_href = publication_repo(repo_href) + update_distributation_pub(dist_href, pub_href) if __name__ == "__main__": -- cgit v1.2.3 From 19e4e254848729b0ebc1ff05514cd6dc91234355 Mon Sep 17 00:00:00 2001 From: Lu Qiuwen Date: Thu, 26 Dec 2019 10:24:47 +0800 Subject: 集成geedge.net的RPM源配置。 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 7 +++++-- netrc.conf | 3 +++ repo.internal.geedge.net.repo | 4 ++++ 3 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 netrc.conf create mode 100644 repo.internal.geedge.net.repo diff --git a/Dockerfile b/Dockerfile index 30a7929..42b640d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,7 @@ FROM centos:7 +COPY rpm_upload_tools.py /root/rpm_upload_tools.py +COPY repo.internal.geedge.net.repo /etc/yum.repos.d/repo.internal.geedge.net.repo +COPY netrc.conf /root/.netrc RUN yum install -y gcc gcc-c++ make libpcap-devel epel-release patch wget socat automake autoconf libtool rpm-build git && \ yum install -y cmake3 && \ @@ -7,6 +10,6 @@ RUN yum install -y gcc gcc-c++ make libpcap-devel epel-release patch wget socat curl -sL https://sentry.io/get-cli/ | bash && \ yum clean all && \ rm -rf /var/cache/yum + rm -rf /root/.netrc -COPY rpm_upload_tools.py /root/rpm_upload_tools.py -CMD [ "/bin/bash" ] \ No newline at end of file +CMD [ "/bin/bash" ] diff --git a/netrc.conf b/netrc.conf new file mode 100644 index 0000000..20815da --- /dev/null +++ b/netrc.conf @@ -0,0 +1,3 @@ +machine repo.internal.geedge.net +login admin +password password \ No newline at end of file diff --git a/repo.internal.geedge.net.repo b/repo.internal.geedge.net.repo new file mode 100644 index 0000000..f2e1b58 --- /dev/null +++ b/repo.internal.geedge.net.repo @@ -0,0 +1,4 @@ +[mesa-platform] +name=mesa-platform +baseurl=http://repo.internal.geedge.net/pulp/content/mesa-platform-stable/ +gpgcheck=0 -- cgit v1.2.3 From b7627b3869e18b98e7868cd9371caee88d64900b Mon Sep 17 00:00:00 2001 From: Lu Qiuwen Date: Thu, 26 Dec 2019 10:26:20 +0800 Subject: 修正Dockerfile的语法错误。 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 42b640d..6ccc9d8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ RUN yum install -y gcc gcc-c++ make libpcap-devel epel-release patch wget socat pip3 install -i https://pypi.tuna.tsinghua.edu.cn/simple httpie && \ curl -sL https://sentry.io/get-cli/ | bash && \ yum clean all && \ - rm -rf /var/cache/yum + rm -rf /var/cache/yum &&\ rm -rf /root/.netrc CMD [ "/bin/bash" ] -- cgit v1.2.3 From a7cab459c9adfa92eef69d38a485f41e49e7c445 Mon Sep 17 00:00:00 2001 From: Lu Qiuwen Date: Thu, 26 Dec 2019 13:24:44 +0800 Subject: 集成.netrc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6ccc9d8..c6932b3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,6 +10,5 @@ RUN yum install -y gcc gcc-c++ make libpcap-devel epel-release patch wget socat curl -sL https://sentry.io/get-cli/ | bash && \ yum clean all && \ rm -rf /var/cache/yum &&\ - rm -rf /root/.netrc CMD [ "/bin/bash" ] -- cgit v1.2.3 From 1a4216acdb4e4c70dfae7542596088fd6d580652 Mon Sep 17 00:00:00 2001 From: Lu Qiuwen Date: Thu, 26 Dec 2019 13:40:01 +0800 Subject: 删去空行修正编译失败问题。 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c6932b3..7b4f817 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,6 +9,6 @@ RUN yum install -y gcc gcc-c++ make libpcap-devel epel-release patch wget socat pip3 install -i https://pypi.tuna.tsinghua.edu.cn/simple httpie && \ curl -sL https://sentry.io/get-cli/ | bash && \ yum clean all && \ - rm -rf /var/cache/yum &&\ + rm -rf /var/cache/yum CMD [ "/bin/bash" ] -- cgit v1.2.3 From 598775bd18a09e370df2258ab075e5296c371a48 Mon Sep 17 00:00:00 2001 From: Lu Qiuwen Date: Thu, 26 Dec 2019 15:10:40 +0800 Subject: 增加上传目录中rpm的功能 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rpm_upload_tools.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/rpm_upload_tools.py b/rpm_upload_tools.py index ddf1c35..28ecf0e 100644 --- a/rpm_upload_tools.py +++ b/rpm_upload_tools.py @@ -196,9 +196,21 @@ def main(): global PULP_SERVER_URL_BASE PULP_SERVER_URL_BASE = args.pulp_server_url + package_list = [] + for package in args.package: + if os.path.isfile(package): + package_list.append(package) + + for file in os.listdir(package): + path = os.path.join(package, file) + if os.path.isfile(path) and path.endswith('.rpm'): + package_list.append(path) + + logging.info('RPMS: %s', str(package_list)) + orphans_cleanup() package_href_collection = [] - for package_path in args.package: + for package_path in package_list: package_basename = os.path.basename(package_path) artifact_href = artifact_create(package_path) package_href = content_create_from_artifact( -- cgit v1.2.3 From 85147e133d0dffe51acb398b14dd0404b3eacbca Mon Sep 17 00:00:00 2001 From: Lu Qiuwen Date: Thu, 26 Dec 2019 16:08:52 +0800 Subject: 修正传入文件时的问题 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rpm_upload_tools.py | 1 + 1 file changed, 1 insertion(+) diff --git a/rpm_upload_tools.py b/rpm_upload_tools.py index 28ecf0e..49a9b2b 100644 --- a/rpm_upload_tools.py +++ b/rpm_upload_tools.py @@ -200,6 +200,7 @@ def main(): for package in args.package: if os.path.isfile(package): package_list.append(package) + continue for file in os.listdir(package): path = os.path.join(package, file) -- cgit v1.2.3