From 83964878e92ce0687d47409b5832e7220e57d79f Mon Sep 17 00:00:00 2001 From: Nate Case Date: Fri, 5 Sep 2008 14:27:31 -0500 Subject: [PATCH] Add XML-RPC interface and command line client Introduce a new XML-RPC Patchwork interface inspired by the SOAP interface from the old Patchwork. The interface itself is fairly lightweight and generic, and provides read-only access to a limited subset of the Patchwork database, along with server-side search and flexible filtering capabilities. The command line client is modeled after the old one with some additional filtering options. The XML-RPC interface is disabled by default. You can enable it by setting ENABLE_XMLRPC = True in local_settings.py This feature uses the django-xmlrpc package available from http://django-xmlrpc.googlecode.com. Signed-off-by: Nate Case Signed-off-by: Jeremy Kerr --- apps/patchwork/bin/pwclient.py | 330 +++++++++++++++++++++++++++++++++ apps/patchwork/xmlrpc.py | 258 ++++++++++++++++++++++++++ apps/settings.py | 20 ++ apps/urls.py | 1 + docs/INSTALL | 16 ++ 5 files changed, 625 insertions(+) create mode 100755 apps/patchwork/bin/pwclient.py create mode 100644 apps/patchwork/xmlrpc.py diff --git a/apps/patchwork/bin/pwclient.py b/apps/patchwork/bin/pwclient.py new file mode 100755 index 0000000..765e66b --- /dev/null +++ b/apps/patchwork/bin/pwclient.py @@ -0,0 +1,330 @@ +#!/usr/bin/env python +# +# Patchwork command line client +# Copyright (C) 2008 Nate Case +# +# This file is part of the Patchwork package. +# +# Patchwork is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# Patchwork is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchwork; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +import os +import sys +import xmlrpclib +import getopt +import string +import tempfile +import subprocess + +# Default Patchwork remote XML-RPC server URL +# This script will check the PW_XMLRPC_URL environment variable +# for the URL to access. If that is unspecified, it will fallback to +# the hardcoded default value specified here. +DEFAULT_URL = "http://patchwork:80/xmlrpc/" + +PW_XMLRPC_URL = os.getenv("PW_XMLRPC_URL") +if not PW_XMLRPC_URL: + PW_XMLRPC_URL = DEFAULT_URL + +class Filter: + """Filter for selecting patches.""" + def __init__(self): + # These fields refer to specific objects, so they are special + # because we have to resolve them to IDs before passing the + # filter to the server + self.state = "" + self.project = "" + + # The dictionary that gets passed to via XML-RPC + self.d = {} + + def add(self, field, value): + if field == 'state': + self.state = value + elif field == 'project': + self.project = value + else: + # OK to add directly + self.d[field] = value + + def resolve_ids(self, rpc): + """Resolve State, Project, and Person IDs based on filter strings.""" + if self.state != "": + id = state_id_by_name(rpc, self.state) + if id == 0: + sys.stderr.write("Note: No State found matching %s*, " \ + "ignoring filter\n" % self.state) + else: + self.d['state_id'] = id + + if self.project != "": + id = project_id_by_name(rpc, self.project) + if id == 0: + sys.stderr.write("Note: No Project found matching %s, " \ + "ignoring filter\n" % self.project) + else: + self.d['project_id'] = id + + def __str__(self): + """Return human-readable description of the filter.""" + return str(self.d) + +def usage(): + sys.stderr.write("Usage: %s [options]\n\n" % \ + (os.path.basename(sys.argv[0]))) + sys.stderr.write("Where is one of:\n") + sys.stderr.write( +""" apply : Apply a patch (in the current dir, using -p1) + get : Download a patch and save it locally + projects : List all projects + states : Show list of potential patch states + list [str] : List patches, using the optional filters specified + below and an optional substring to search for patches + by name + search [str] : Same as 'list' + view : View a patch\n""") + sys.stderr.write("""\nFilter options for 'list' and 'search': + -s : Filter by patch state (e.g., 'New', 'Accepted', etc.) + -p : Filter by project name (see 'projects' for list) + -w : Filter by submitter (name, e-mail substring search) + -d : Filter by delegate (name, e-mail substring search) + -n : Restrict number of results\n""") + sys.exit(1) + +def project_id_by_name(rpc, linkname): + """Given a project short name, look up the Project ID.""" + if len(linkname) == 0: + return 0 + # The search requires - instead of _ + search = linkname.replace("_", "-") + projects = rpc.project_list(search, 0) + for project in projects: + if project['linkname'].replace("_", "-") == search: + return project['id'] + return 0 + +def state_id_by_name(rpc, name): + """Given a partial state name, look up the state ID.""" + if len(name) == 0: + return 0 + states = rpc.state_list(name, 0) + for state in states: + if state['name'].lower().startswith(name.lower()): + return state['id'] + return 0 + +def person_ids_by_name(rpc, name): + """Given a partial name or email address, return a list of the + person IDs that match.""" + if len(name) == 0: + return [] + people = rpc.person_list(name, 0) + return map(lambda x: x['id'], people) + +def list_patches(patches): + """Dump a list of patches to stdout.""" + print("%-5s %-12s %s" % ("ID", "State", "Name")) + print("%-5s %-12s %s" % ("--", "-----", "----")) + for patch in patches: + print("%-5d %-12s %s" % (patch['id'], patch['state'], patch['name'])) + +def action_list(rpc, filter, submitter_str, delegate_str): + filter.resolve_ids(rpc) + + if submitter_str != "": + ids = person_ids_by_name(rpc, submitter_str) + if len(ids) == 0: + sys.stderr.write("Note: Nobody found matching *%s*\n", \ + submitter_str) + else: + for id in ids: + person = rpc.person_get(id) + print "Patches submitted by %s <%s>:" % \ + (person['name'], person['email']) + f = filter + f.add("submitter_id", id) + patches = rpc.patch_list(f.d) + list_patches(patches) + return + + if delegate_str != "": + ids = person_ids_by_name(rpc, delegate_str) + if len(ids) == 0: + sys.stderr.write("Note: Nobody found matching *%s*\n", \ + delegate_str) + else: + for id in ids: + person = rpc.person_get(id) + print "Patches delegated to %s <%s>:" % \ + (person['name'], person['email']) + f = filter + f.add("delegate_id", id) + patches = rpc.patch_list(f.d) + list_patches(patches) + return + + patches = rpc.patch_list(filter.d) + list_patches(patches) + +def action_projects(rpc): + projects = rpc.project_list("", 0) + print("%-5s %-24s %s" % ("ID", "Name", "Description")) + print("%-5s %-24s %s" % ("--", "----", "-----------")) + for project in projects: + print("%-5d %-24s %s" % (project['id'], \ + project['linkname'].replace("_", "-"), \ + project['name'])) + +def action_states(rpc): + states = rpc.state_list("", 0) + print("%-5s %s" % ("ID", "Name")) + print("%-5s %s" % ("--", "----")) + for state in states: + print("%-5d %s" % (state['id'], state['name'])) + +def action_get(rpc, patch_id): + patch = rpc.patch_get(patch_id) + s = rpc.patch_get_mbox(patch_id) + + if patch == {} or len(s) == 0: + sys.stderr.write("Unable to get patch %d\n" % patch_id) + sys.exit(1) + + base_fname = fname = os.path.basename(patch['filename']) + i = 0 + while os.path.exists(fname): + fname = "%s.%d" % (base_fname, i) + i += 1 + + try: + f = open(fname, "w") + except: + sys.stderr.write("Unable to open %s for writing\n" % fname) + sys.exit(1) + + try: + f.write(s) + f.close() + print "Saved patch to %s" % fname + except: + sys.stderr.write("Failed to write to %s\n" % fname) + sys.exit(1) + +def action_apply(rpc, patch_id): + patch = rpc.patch_get(patch_id) + if patch == {}: + sys.stderr.write("Error getting information on patch ID %d\n" % \ + patch_id) + sys.exit(1) + print "Applying patch #%d to current directory" % patch_id + print "Description: %s" % patch['name'] + s = rpc.patch_get_mbox(patch_id) + if len(s) > 0: + proc = subprocess.Popen(['patch', '-p1'], stdin = subprocess.PIPE) + proc.communicate(s) + else: + sys.stderr.write("Error: No patch content found\n") + sys.exit(1) + +def main(): + try: + opts, args = getopt.getopt(sys.argv[2:], 's:p:w:d:n:') + except getopt.GetoptError, err: + print str(err) + usage() + + if len(sys.argv) < 2: + usage() + + action = sys.argv[1].lower() + + filt = Filter() + submitter_str = "" + delegate_str = "" + + for name, value in opts: + if name == '-s': + filt.add("state", value) + elif name == '-p': + filt.add("project", value) + elif name == '-w': + submitter_str = value + elif name == '-d': + delegate_str = value + elif name == '-n': + try: + filt.add("max_count", int(value)) + except: + sys.stderr.write("Invalid maximum count '%s'\n" % value) + usage() + else: + sys.stderr.write("Unknown option '%s'\n" % name) + usage() + + if len(args) > 1: + sys.stderr.write("Too many arguments specified\n") + usage() + + try: + rpc = xmlrpclib.Server(PW_XMLRPC_URL) + except: + sys.stderr.write("Unable to connect to %s\n" % PW_XMLRPC_URL) + sys.exit(1) + + if action == 'list' or action == 'search': + if len(args) > 0: + filt.add("name__icontains", args[0]) + action_list(rpc, filt, submitter_str, delegate_str) + + elif action.startswith('project'): + action_projects(rpc) + + elif action.startswith('state'): + action_states(rpc) + + elif action == 'view': + try: + patch_id = int(args[0]) + except: + sys.stderr.write("Invalid patch ID given\n") + sys.exit(1) + + s = rpc.patch_get_mbox(patch_id) + if len(s) > 0: + print s + + elif action == 'get' or action == 'save': + try: + patch_id = int(args[0]) + except: + sys.stderr.write("Invalid patch ID given\n") + sys.exit(1) + + action_get(rpc, patch_id) + + elif action == 'apply': + try: + patch_id = int(args[0]) + except: + sys.stderr.write("Invalid patch ID given\n") + sys.exit(1) + + action_apply(rpc, patch_id) + + else: + sys.stderr.write("Unknown action '%s'\n" % action) + usage() + +if __name__ == "__main__": + main() diff --git a/apps/patchwork/xmlrpc.py b/apps/patchwork/xmlrpc.py new file mode 100644 index 0000000..a661611 --- /dev/null +++ b/apps/patchwork/xmlrpc.py @@ -0,0 +1,258 @@ +# Patchwork - automated patch tracking system +# Copyright (C) 2008 Nate Case +# +# This file is part of the Patchwork package. +# +# Patchwork is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# Patchwork is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchwork; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# The XML-RPC interface provides a watered down, read-only interface to +# the Patchwork database. It's intended to be safe to export to the public +# Internet. A small subset of the object data is included, and the type +# of requests/queries you can do is limited by the methods +# that we export. + +from patchwork.models import Patch, Project, Person, Bundle, State + +# We allow most of the Django field lookup types for remote queries +LOOKUP_TYPES = ["iexact", "contains", "icontains", "gt", "gte", "lt", + "in", "startswith", "istartswith", "endswith", + "iendswith", "range", "year", "month", "day", "isnull" ] + +####################################################################### +# Helper functions +####################################################################### + +def project_to_dict(obj): + """Return a trimmed down dictionary representation of a Project + object which is OK to send to the client.""" + return \ + { + 'id' : obj.id, + 'linkname' : obj.linkname, + 'name' : obj.name, + } + +def person_to_dict(obj): + """Return a trimmed down dictionary representation of a Person + object which is OK to send to the client.""" + return \ + { + 'id' : obj.id, + 'email' : obj.email, + 'name' : obj.name, + 'user' : str(obj.user), + } + +def patch_to_dict(obj): + """Return a trimmed down dictionary representation of a Patch + object which is OK to send to the client.""" + return \ + { + 'id' : obj.id, + 'date' : str(obj.date), + 'filename' : obj.filename(), + 'msgid' : obj.msgid, + 'name' : obj.name, + 'project' : str(obj.project), + 'project_id' : obj.project_id, + 'state' : str(obj.state), + 'state_id' : obj.state_id, + 'submitter' : str(obj.submitter), + 'submitter_id' : obj.submitter_id, + 'delegate' : str(obj.delegate), + 'delegate_id' : max(obj.delegate_id, 0), + 'commit_ref' : max(obj.commit_ref, ''), + } + +def bundle_to_dict(obj): + """Return a trimmed down dictionary representation of a Bundle + object which is OK to send to the client.""" + return \ + { + 'id' : obj.id, + 'name' : obj.name, + 'n_patches' : obj.n_patches(), + 'public_url' : obj.public_url(), + } + +def state_to_dict(obj): + """Return a trimmed down dictionary representation of a State + object which is OK to send to the client.""" + return \ + { + 'id' : obj.id, + 'name' : obj.name, + } + +####################################################################### +# Public XML-RPC methods +####################################################################### + +def pw_rpc_version(): + """Return Patchwork XML-RPC interface version.""" + return 1 + +def project_list(search_str="", max_count=0): + """Get a list of projects matching the given filters.""" + try: + if len(search_str) > 0: + projects = Project.objects.filter(name__icontains = search_str) + else: + projects = Project.objects.all() + + if max_count > 0: + return map(project_to_dict, projects)[:max_count] + else: + return map(project_to_dict, projects) + except: + return [] + +def project_get(project_id): + """Return structure for the given project ID.""" + try: + project = Project.objects.filter(id = project_id)[0] + return project_to_dict(project) + except: + return {} + +def person_list(search_str="", max_count=0): + """Get a list of Person objects matching the given filters.""" + try: + if len(search_str) > 0: + people = (Person.objects.filter(name__icontains = search_str) | + Person.objects.filter(email__icontains = search_str)) + else: + people = Person.objects.all() + + if max_count > 0: + return map(person_to_dict, people)[:max_count] + else: + return map(person_to_dict, people) + + except: + return [] + +def person_get(person_id): + """Return structure for the given person ID.""" + try: + person = Person.objects.filter(id = person_id)[0] + return person_to_dict(person) + except: + return {} + +def patch_list(filter={}): + """Get a list of patches matching the given filters.""" + try: + # We allow access to many of the fields. But, some fields are + # filtered by raw object so we must lookup by ID instead over + # XML-RPC. + ok_fields = [ + "id", + "name", + "project_id", + "submitter_id", + "delegate_id", + "state_id", + "date", + "commit_ref", + "hash", + "msgid", + "name", + "max_count", + ] + + dfilter = {} + max_count = 0 + + for key in filter: + parts = key.split("__") + if ok_fields.count(parts[0]) == 0: + # Invalid field given + return [] + if len(parts) > 1: + if LOOKUP_TYPES.count(parts[1]) == 0: + # Invalid lookup type given + return [] + + if parts[0] == 'project_id': + dfilter['project'] = Project.objects.filter(id = + filter[key])[0] + elif parts[0] == 'submitter_id': + dfilter['submitter'] = Person.objects.filter(id = + filter[key])[0] + elif parts[0] == 'state_id': + dfilter['state'] = State.objects.filter(id = + filter[key])[0] + elif parts[0] == 'max_count': + max_count = filter[key] + else: + dfilter[key] = filter[key] + + patches = Patch.objects.filter(**dfilter) + + if max_count > 0: + return map(patch_to_dict, patches)[:max_count] + else: + return map(patch_to_dict, patches) + + except: + return [] + +def patch_get(patch_id): + """Return structure for the given patch ID.""" + try: + patch = Patch.objects.filter(id = patch_id)[0] + return patch_to_dict(patch) + except: + return {} + +def patch_get_mbox(patch_id): + """Return mbox string for the given patch ID.""" + try: + patch = Patch.objects.filter(id = patch_id)[0] + return patch.mbox().as_string() + except: + return "" + +def patch_get_diff(patch_id): + """Return diff for the given patch ID.""" + try: + patch = Patch.objects.filter(id = patch_id)[0] + return patch.content + except: + return "" + +def state_list(search_str="", max_count=0): + """Get a list of state structures matching the given search string.""" + try: + if len(search_str) > 0: + states = State.objects.filter(name__icontains = search_str) + else: + states = State.objects.all() + + if max_count > 0: + return map(state_to_dict, states)[:max_count] + else: + return map(state_to_dict, states) + except: + return [] + +def state_get(state_id): + """Return structure for the given state ID.""" + try: + state = State.objects.filter(id = state_id)[0] + return state_to_dict(state) + except: + return {} diff --git a/apps/settings.py b/apps/settings.py index d5fd7b1..f70ac2a 100644 --- a/apps/settings.py +++ b/apps/settings.py @@ -98,8 +98,28 @@ DEFAULT_FROM_EMAIL = 'Patchwork ' ACCOUNT_ACTIVATION_DAYS = 7 +# Set to True to enable the Patchwork XML-RPC interface +ENABLE_XMLRPC = False + +XMLRPC_METHODS = ( + # List methods to be exposed in the form (, ,) + ('patchwork.xmlrpc.pw_rpc_version', 'pw_rpc_version',), + ('patchwork.xmlrpc.patch_list', 'patch_list',), + ('patchwork.xmlrpc.patch_get', 'patch_get',), + ('patchwork.xmlrpc.patch_get_mbox', 'patch_get_mbox',), + ('patchwork.xmlrpc.patch_get_diff', 'patch_get_diff',), + ('patchwork.xmlrpc.project_list', 'project_list',), + ('patchwork.xmlrpc.project_get', 'project_get',), + ('patchwork.xmlrpc.person_list', 'person_list',), + ('patchwork.xmlrpc.person_get', 'person_get',), + ('patchwork.xmlrpc.state_list', 'state_list',), + ('patchwork.xmlrpc.state_get', 'state_get',), +) + try: from local_settings import * + if ENABLE_XMLRPC: + INSTALLED_APPS = INSTALLED_APPS + ('django_xmlrpc',) except ImportError, ex: import sys sys.stderr.write(\ diff --git a/apps/urls.py b/apps/urls.py index 6e7f34e..3ccdd1a 100644 --- a/apps/urls.py +++ b/apps/urls.py @@ -46,4 +46,5 @@ urlpatterns = patterns('', {'document_root': '/srv/patchwork/htdocs/js'}), (r'^images/(?P.*)$', 'django.views.static.serve', {'document_root': '/srv/patchwork/htdocs/images'}), + (r'xmlrpc/$', 'django_xmlrpc.views.handle_xmlrpc',), ) diff --git a/docs/INSTALL b/docs/INSTALL index 1748601..cba69d5 100644 --- a/docs/INSTALL +++ b/docs/INSTALL @@ -62,6 +62,17 @@ in brackets): cd ../../apps ln -s ../lib/packages/django-registration ./registration + (OPTIONAL) If you want to enable the Patchwork XML-RPC interface, + which is required for pwclient to work, you'll need to set up the + django_xmlrpc package: + + cd lib/packages/ + wget \ + http://django-xmlrpc.googlecode.com/files/django_xmlrpc-0.1.tar.gz + tar -zxf django_xmlrpc-0.1.tar.gz + cd ../../apps + ln -s ../lib/packages/django_xmlrpc ./django_xmlrpc + The settings.py file contains default settings for patchwork, you'll need to configure settings for your own setup. @@ -86,6 +97,11 @@ in brackets): MEDIA_ROOT TEMPLATE_DIRS + If you wish to enable the XML-RPC interface, add the following to + your local_settings.py file: + + ENABLE_XMLRPC = True + Then, get patchwork to create its tables in your configured database: cd apps/ -- 2.39.2