]> git.ozlabs.org Git - patchwork/commitdiff
Add XML-RPC interface and command line client
authorNate Case <ncase@xes-inc.com>
Fri, 5 Sep 2008 19:27:31 +0000 (14:27 -0500)
committerJeremy Kerr <jk@ozlabs.org>
Mon, 8 Sep 2008 00:36:15 +0000 (10:36 +1000)
Introduce a new XML-RPC Patchwork interface inspired by the SOAP
interface from the old Patchwork.  The interface itself is fairly
lightweight and generic, and provides read-only access to a limited
subset of the Patchwork database, along with server-side search
and flexible filtering capabilities.

The command line client is modeled after the old one with some
additional filtering options.

The XML-RPC interface is disabled by default.  You can enable it
by setting ENABLE_XMLRPC = True in local_settings.py

This feature uses the django-xmlrpc package available from
http://django-xmlrpc.googlecode.com.

Signed-off-by: Nate Case <ncase@xes-inc.com>
Signed-off-by: Jeremy Kerr <jk@ozlabs.org>
apps/patchwork/bin/pwclient.py [new file with mode: 0755]
apps/patchwork/xmlrpc.py [new file with mode: 0644]
apps/settings.py
apps/urls.py
docs/INSTALL

diff --git a/apps/patchwork/bin/pwclient.py b/apps/patchwork/bin/pwclient.py
new file mode 100755 (executable)
index 0000000..765e66b
--- /dev/null
@@ -0,0 +1,330 @@
+#!/usr/bin/env python
+#
+# Patchwork command line client
+# Copyright (C) 2008 Nate Case <ncase@xes-inc.com>
+#
+# This file is part of the Patchwork package.
+#
+# Patchwork is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# Patchwork is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Patchwork; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+import os
+import sys
+import xmlrpclib
+import getopt
+import string
+import tempfile
+import subprocess
+
+# Default Patchwork remote XML-RPC server URL
+# This script will check the PW_XMLRPC_URL environment variable
+# for the URL to access.  If that is unspecified, it will fallback to
+# the hardcoded default value specified here.
+DEFAULT_URL = "http://patchwork:80/xmlrpc/"
+
+PW_XMLRPC_URL = os.getenv("PW_XMLRPC_URL")
+if not PW_XMLRPC_URL:
+    PW_XMLRPC_URL = DEFAULT_URL
+
+class Filter:
+    """Filter for selecting patches."""
+    def __init__(self):
+        # These fields refer to specific objects, so they are special
+        # because we have to resolve them to IDs before passing the
+        # filter to the server
+        self.state = ""
+        self.project = ""
+
+        # The dictionary that gets passed to via XML-RPC
+        self.d = {}
+
+    def add(self, field, value):
+        if field == 'state':
+            self.state = value
+        elif field == 'project':
+            self.project = value
+        else:
+            # OK to add directly
+            self.d[field] = value
+
+    def resolve_ids(self, rpc):
+        """Resolve State, Project, and Person IDs based on filter strings."""
+        if self.state != "":
+            id = state_id_by_name(rpc, self.state)
+            if id == 0:
+                sys.stderr.write("Note: No State found matching %s*, " \
+                                 "ignoring filter\n" % self.state)
+            else:
+                self.d['state_id'] = id
+
+        if self.project != "":
+            id = project_id_by_name(rpc, self.project)
+            if id == 0:
+                sys.stderr.write("Note: No Project found matching %s, " \
+                                 "ignoring filter\n" % self.project)
+            else:
+                self.d['project_id'] = id
+
+    def __str__(self):
+        """Return human-readable description of the filter."""
+        return str(self.d)
+
+def usage():
+    sys.stderr.write("Usage: %s <action> [options]\n\n" % \
+                        (os.path.basename(sys.argv[0])))
+    sys.stderr.write("Where <action> is one of:\n")
+    sys.stderr.write(
+"""        apply <ID>    : Apply a patch (in the current dir, using -p1)
+        get <ID>      : Download a patch and save it locally
+        projects      : List all projects
+        states        : Show list of potential patch states
+        list [str]    : List patches, using the optional filters specified
+                        below and an optional substring to search for patches
+                        by name
+        search [str]  : Same as 'list'
+        view <ID>     : View a patch\n""")
+    sys.stderr.write("""\nFilter options for 'list' and 'search':
+        -s <state>    : Filter by patch state (e.g., 'New', 'Accepted', etc.)
+        -p <project>  : Filter by project name (see 'projects' for list)
+        -w <who>      : Filter by submitter (name, e-mail substring search)
+        -d <who>      : Filter by delegate (name, e-mail substring search)
+        -n <max #>    : Restrict number of results\n""")
+    sys.exit(1)
+
+def project_id_by_name(rpc, linkname):
+    """Given a project short name, look up the Project ID."""
+    if len(linkname) == 0:
+        return 0
+    # The search requires - instead of _
+    search = linkname.replace("_", "-")
+    projects = rpc.project_list(search, 0)
+    for project in projects:
+        if project['linkname'].replace("_", "-") == search:
+            return project['id']
+    return 0
+
+def state_id_by_name(rpc, name):
+    """Given a partial state name, look up the state ID."""
+    if len(name) == 0:
+        return 0
+    states = rpc.state_list(name, 0)
+    for state in states:
+        if state['name'].lower().startswith(name.lower()):
+            return state['id']
+    return 0
+
+def person_ids_by_name(rpc, name):
+    """Given a partial name or email address, return a list of the
+    person IDs that match."""
+    if len(name) == 0:
+        return []
+    people = rpc.person_list(name, 0)
+    return map(lambda x: x['id'], people)
+
+def list_patches(patches):
+    """Dump a list of patches to stdout."""
+    print("%-5s %-12s %s" % ("ID", "State", "Name"))
+    print("%-5s %-12s %s" % ("--", "-----", "----"))
+    for patch in patches:
+        print("%-5d %-12s %s" % (patch['id'], patch['state'], patch['name']))
+
+def action_list(rpc, filter, submitter_str, delegate_str):
+    filter.resolve_ids(rpc)
+
+    if submitter_str != "":
+        ids = person_ids_by_name(rpc, submitter_str)
+        if len(ids) == 0:
+            sys.stderr.write("Note: Nobody found matching *%s*\n", \
+                             submitter_str)
+        else:
+            for id in ids:
+                person = rpc.person_get(id)
+                print "Patches submitted by %s <%s>:" % \
+                        (person['name'], person['email'])
+                f = filter
+                f.add("submitter_id", id)
+                patches = rpc.patch_list(f.d)
+                list_patches(patches)
+        return
+
+    if delegate_str != "":
+        ids = person_ids_by_name(rpc, delegate_str)
+        if len(ids) == 0:
+            sys.stderr.write("Note: Nobody found matching *%s*\n", \
+                             delegate_str)
+        else:
+            for id in ids:
+                person = rpc.person_get(id)
+                print "Patches delegated to %s <%s>:" % \
+                        (person['name'], person['email'])
+                f = filter
+                f.add("delegate_id", id)
+                patches = rpc.patch_list(f.d)
+                list_patches(patches)
+        return
+
+    patches = rpc.patch_list(filter.d)
+    list_patches(patches)
+
+def action_projects(rpc):
+    projects = rpc.project_list("", 0)
+    print("%-5s %-24s %s" % ("ID", "Name", "Description"))
+    print("%-5s %-24s %s" % ("--", "----", "-----------"))
+    for project in projects:
+        print("%-5d %-24s %s" % (project['id'], \
+                project['linkname'].replace("_", "-"), \
+                project['name']))
+
+def action_states(rpc):
+    states = rpc.state_list("", 0)
+    print("%-5s %s" % ("ID", "Name"))
+    print("%-5s %s" % ("--", "----"))
+    for state in states:
+        print("%-5d %s" % (state['id'], state['name']))
+
+def action_get(rpc, patch_id):
+    patch = rpc.patch_get(patch_id)
+    s = rpc.patch_get_mbox(patch_id)
+
+    if patch == {} or len(s) == 0:
+        sys.stderr.write("Unable to get patch %d\n" % patch_id)
+        sys.exit(1)
+
+    base_fname = fname = os.path.basename(patch['filename'])
+    i = 0
+    while os.path.exists(fname):
+        fname = "%s.%d" % (base_fname, i)
+        i += 1
+
+    try:
+        f = open(fname, "w")
+    except:
+        sys.stderr.write("Unable to open %s for writing\n" % fname)
+        sys.exit(1)
+
+    try:
+        f.write(s)
+        f.close()
+        print "Saved patch to %s" % fname
+    except:
+        sys.stderr.write("Failed to write to %s\n" % fname)
+        sys.exit(1)
+
+def action_apply(rpc, patch_id):
+    patch = rpc.patch_get(patch_id)
+    if patch == {}:
+        sys.stderr.write("Error getting information on patch ID %d\n" % \
+                         patch_id)
+        sys.exit(1)
+    print "Applying patch #%d to current directory" % patch_id
+    print "Description: %s" % patch['name']
+    s = rpc.patch_get_mbox(patch_id)
+    if len(s) > 0:
+        proc = subprocess.Popen(['patch', '-p1'], stdin = subprocess.PIPE)
+        proc.communicate(s)
+    else:
+        sys.stderr.write("Error: No patch content found\n")
+        sys.exit(1)
+
+def main():
+    try:
+        opts, args = getopt.getopt(sys.argv[2:], 's:p:w:d:n:')
+    except getopt.GetoptError, err:
+        print str(err)
+        usage()
+
+    if len(sys.argv) < 2:
+        usage()
+
+    action = sys.argv[1].lower()
+
+    filt = Filter()
+    submitter_str = ""
+    delegate_str = ""
+
+    for name, value in opts:
+        if name == '-s':
+            filt.add("state", value)
+        elif name == '-p':
+            filt.add("project", value)
+        elif name == '-w':
+            submitter_str = value
+        elif name == '-d':
+            delegate_str = value
+        elif name == '-n':
+            try:
+                filt.add("max_count", int(value))
+            except:
+                sys.stderr.write("Invalid maximum count '%s'\n" % value)
+                usage()
+        else:
+            sys.stderr.write("Unknown option '%s'\n" % name)
+            usage()
+
+    if len(args) > 1:
+        sys.stderr.write("Too many arguments specified\n")
+        usage()
+
+    try:
+        rpc = xmlrpclib.Server(PW_XMLRPC_URL)
+    except:
+        sys.stderr.write("Unable to connect to %s\n" % PW_XMLRPC_URL)
+        sys.exit(1)
+
+    if action == 'list' or action == 'search':
+        if len(args) > 0:
+            filt.add("name__icontains", args[0])
+        action_list(rpc, filt, submitter_str, delegate_str)
+
+    elif action.startswith('project'):
+        action_projects(rpc)
+
+    elif action.startswith('state'):
+        action_states(rpc)
+
+    elif action == 'view':
+        try:
+            patch_id = int(args[0])
+        except:
+            sys.stderr.write("Invalid patch ID given\n")
+            sys.exit(1)
+
+        s = rpc.patch_get_mbox(patch_id)
+        if len(s) > 0:
+            print s
+
+    elif action == 'get' or action == 'save':
+        try:
+            patch_id = int(args[0])
+        except:
+            sys.stderr.write("Invalid patch ID given\n")
+            sys.exit(1)
+
+        action_get(rpc, patch_id)
+
+    elif action == 'apply':
+        try:
+            patch_id = int(args[0])
+        except:
+            sys.stderr.write("Invalid patch ID given\n")
+            sys.exit(1)
+
+        action_apply(rpc, patch_id)
+
+    else:
+        sys.stderr.write("Unknown action '%s'\n" % action)
+        usage()
+
+if __name__ == "__main__":
+    main()
diff --git a/apps/patchwork/xmlrpc.py b/apps/patchwork/xmlrpc.py
new file mode 100644 (file)
index 0000000..a661611
--- /dev/null
@@ -0,0 +1,258 @@
+# Patchwork - automated patch tracking system
+# Copyright (C) 2008 Nate Case <ncase@xes-inc.com>
+#
+# This file is part of the Patchwork package.
+#
+# Patchwork is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# Patchwork is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Patchwork; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#
+# The XML-RPC interface provides a watered down, read-only interface to
+# the Patchwork database.  It's intended to be safe to export to the public
+# Internet.  A small subset of the object data is included, and the type
+# of requests/queries you can do is limited by the methods
+# that we export.
+
+from patchwork.models import Patch, Project, Person, Bundle, State
+
+# We allow most of the Django field lookup types for remote queries
+LOOKUP_TYPES = ["iexact", "contains", "icontains", "gt", "gte", "lt",
+                "in", "startswith", "istartswith", "endswith",
+                "iendswith", "range", "year", "month", "day", "isnull" ]
+
+#######################################################################
+# Helper functions
+#######################################################################
+
+def project_to_dict(obj):
+    """Return a trimmed down dictionary representation of a Project
+    object which is OK to send to the client."""
+    return \
+        {
+         'id'           : obj.id,
+         'linkname'     : obj.linkname,
+         'name'         : obj.name,
+        }
+
+def person_to_dict(obj):
+    """Return a trimmed down dictionary representation of a Person
+    object which is OK to send to the client."""
+    return \
+        {
+         'id'           : obj.id,
+         'email'        : obj.email,
+         'name'         : obj.name,
+         'user'         : str(obj.user),
+        }
+
+def patch_to_dict(obj):
+    """Return a trimmed down dictionary representation of a Patch
+    object which is OK to send to the client."""
+    return \
+        {
+         'id'           : obj.id,
+         'date'         : str(obj.date),
+         'filename'     : obj.filename(),
+         'msgid'        : obj.msgid,
+         'name'         : obj.name,
+         'project'      : str(obj.project),
+         'project_id'   : obj.project_id,
+         'state'        : str(obj.state),
+         'state_id'     : obj.state_id,
+         'submitter'    : str(obj.submitter),
+         'submitter_id' : obj.submitter_id,
+         'delegate'     : str(obj.delegate),
+         'delegate_id'  : max(obj.delegate_id, 0),
+         'commit_ref'   : max(obj.commit_ref, ''),
+        }
+
+def bundle_to_dict(obj):
+    """Return a trimmed down dictionary representation of a Bundle
+    object which is OK to send to the client."""
+    return \
+        {
+         'id'           : obj.id,
+         'name'         : obj.name,
+         'n_patches'    : obj.n_patches(),
+         'public_url'   : obj.public_url(),
+        }
+
+def state_to_dict(obj):
+    """Return a trimmed down dictionary representation of a State
+    object which is OK to send to the client."""
+    return \
+        {
+         'id'           : obj.id,
+         'name'         : obj.name,
+        }
+
+#######################################################################
+# Public XML-RPC methods
+#######################################################################
+
+def pw_rpc_version():
+    """Return Patchwork XML-RPC interface version."""
+    return 1
+
+def project_list(search_str="", max_count=0):
+    """Get a list of projects matching the given filters."""
+    try:
+        if len(search_str) > 0:
+            projects = Project.objects.filter(name__icontains = search_str)
+        else:
+            projects = Project.objects.all()
+
+        if max_count > 0:
+            return map(project_to_dict, projects)[:max_count]
+        else:
+            return map(project_to_dict, projects)
+    except:
+        return []
+
+def project_get(project_id):
+    """Return structure for the given project ID."""
+    try:
+        project = Project.objects.filter(id = project_id)[0]
+        return project_to_dict(project)
+    except:
+        return {}
+
+def person_list(search_str="", max_count=0):
+    """Get a list of Person objects matching the given filters."""
+    try:
+        if len(search_str) > 0:
+            people = (Person.objects.filter(name__icontains = search_str) |
+                Person.objects.filter(email__icontains = search_str))
+        else:
+            people = Person.objects.all()
+
+        if max_count > 0:
+            return map(person_to_dict, people)[:max_count]
+        else:
+            return map(person_to_dict, people)
+
+    except:
+        return []
+
+def person_get(person_id):
+    """Return structure for the given person ID."""
+    try:
+        person = Person.objects.filter(id = person_id)[0]
+        return person_to_dict(person)
+    except:
+        return {}
+
+def patch_list(filter={}):
+    """Get a list of patches matching the given filters."""
+    try:
+        # We allow access to many of the fields.  But, some fields are
+        # filtered by raw object so we must lookup by ID instead over
+        # XML-RPC.
+        ok_fields = [
+            "id",
+            "name",
+            "project_id",
+            "submitter_id",
+            "delegate_id",
+            "state_id",
+            "date",
+            "commit_ref",
+            "hash",
+            "msgid",
+            "name",
+            "max_count",
+            ]
+
+        dfilter = {}
+        max_count = 0
+
+        for key in filter:
+            parts = key.split("__")
+            if ok_fields.count(parts[0]) == 0:
+                # Invalid field given
+                return []
+            if len(parts) > 1:
+                if LOOKUP_TYPES.count(parts[1]) == 0:
+                    # Invalid lookup type given
+                    return []
+
+            if parts[0] == 'project_id':
+                dfilter['project'] = Project.objects.filter(id =
+                                        filter[key])[0]
+            elif parts[0] == 'submitter_id':
+                dfilter['submitter'] = Person.objects.filter(id =
+                                        filter[key])[0]
+            elif parts[0] == 'state_id':
+                dfilter['state'] = State.objects.filter(id =
+                                        filter[key])[0]
+            elif parts[0] == 'max_count':
+                max_count = filter[key]
+            else:
+                dfilter[key] = filter[key]
+
+        patches = Patch.objects.filter(**dfilter)
+
+        if max_count > 0:
+            return map(patch_to_dict, patches)[:max_count]
+        else:
+            return map(patch_to_dict, patches)
+
+    except:
+        return []
+
+def patch_get(patch_id):
+    """Return structure for the given patch ID."""
+    try:
+        patch = Patch.objects.filter(id = patch_id)[0]
+        return patch_to_dict(patch)
+    except:
+        return {}
+
+def patch_get_mbox(patch_id):
+    """Return mbox string for the given patch ID."""
+    try:
+        patch = Patch.objects.filter(id = patch_id)[0]
+        return patch.mbox().as_string()
+    except:
+        return ""
+
+def patch_get_diff(patch_id):
+    """Return diff for the given patch ID."""
+    try:
+        patch = Patch.objects.filter(id = patch_id)[0]
+        return patch.content
+    except:
+        return ""
+
+def state_list(search_str="", max_count=0):
+    """Get a list of state structures matching the given search string."""
+    try:
+        if len(search_str) > 0:
+            states = State.objects.filter(name__icontains = search_str)
+        else:
+            states = State.objects.all()
+
+        if max_count > 0:
+            return map(state_to_dict, states)[:max_count]
+        else:
+            return map(state_to_dict, states)
+    except:
+        return []
+
+def state_get(state_id):
+    """Return structure for the given state ID."""
+    try:
+        state = State.objects.filter(id = state_id)[0]
+        return state_to_dict(state)
+    except:
+        return {}
index d5fd7b1a7509c87a8def367589ba46cd055824d3..f70ac2a5a14e0a09fd9ac7fa217a90d61c39e65d 100644 (file)
@@ -98,8 +98,28 @@ DEFAULT_FROM_EMAIL = 'Patchwork <patchwork@patchwork.example.com>'
 
 ACCOUNT_ACTIVATION_DAYS = 7
 
 
 ACCOUNT_ACTIVATION_DAYS = 7
 
+# Set to True to enable the Patchwork XML-RPC interface
+ENABLE_XMLRPC = False
+
+XMLRPC_METHODS = (
+    # List methods to be exposed in the form (<method path>, <xml-rpcname>,)
+    ('patchwork.xmlrpc.pw_rpc_version', 'pw_rpc_version',),
+    ('patchwork.xmlrpc.patch_list',     'patch_list',),
+    ('patchwork.xmlrpc.patch_get',      'patch_get',),
+    ('patchwork.xmlrpc.patch_get_mbox', 'patch_get_mbox',),
+    ('patchwork.xmlrpc.patch_get_diff', 'patch_get_diff',),
+    ('patchwork.xmlrpc.project_list',   'project_list',),
+    ('patchwork.xmlrpc.project_get',    'project_get',),
+    ('patchwork.xmlrpc.person_list',    'person_list',),
+    ('patchwork.xmlrpc.person_get',     'person_get',),
+    ('patchwork.xmlrpc.state_list',     'state_list',),
+    ('patchwork.xmlrpc.state_get',      'state_get',),
+)
+
 try:
     from local_settings import *
 try:
     from local_settings import *
+    if ENABLE_XMLRPC:
+        INSTALLED_APPS = INSTALLED_APPS + ('django_xmlrpc',)
 except ImportError, ex:
     import sys
     sys.stderr.write(\
 except ImportError, ex:
     import sys
     sys.stderr.write(\
index 6e7f34ef6ffbecbaceede041278341f8aca791d1..3ccdd1aaae0213c79243e32444ff6b49e35b7081 100644 (file)
@@ -46,4 +46,5 @@ urlpatterns = patterns('',
         {'document_root': '/srv/patchwork/htdocs/js'}),
      (r'^images/(?P<path>.*)$', 'django.views.static.serve',
         {'document_root': '/srv/patchwork/htdocs/images'}),
         {'document_root': '/srv/patchwork/htdocs/js'}),
      (r'^images/(?P<path>.*)$', 'django.views.static.serve',
         {'document_root': '/srv/patchwork/htdocs/images'}),
+     (r'xmlrpc/$', 'django_xmlrpc.views.handle_xmlrpc',),
 )
 )
index 1748601954c04cc095318f1cfde346433c94d4b8..cba69d55d664d532a13245224d8cba1ab824f25e 100644 (file)
@@ -62,6 +62,17 @@ in brackets):
          cd ../../apps
          ln -s ../lib/packages/django-registration ./registration
 
          cd ../../apps
          ln -s ../lib/packages/django-registration ./registration
 
+       (OPTIONAL) If you want to enable the Patchwork XML-RPC interface,
+        which is required for pwclient to work, you'll need to set up the
+        django_xmlrpc package:
+
+         cd lib/packages/
+         wget \
+             http://django-xmlrpc.googlecode.com/files/django_xmlrpc-0.1.tar.gz
+         tar -zxf django_xmlrpc-0.1.tar.gz
+         cd ../../apps
+         ln -s ../lib/packages/django_xmlrpc ./django_xmlrpc
+
        The settings.py file contains default settings for patchwork, you'll
        need to configure settings for your own setup.
 
        The settings.py file contains default settings for patchwork, you'll
        need to configure settings for your own setup.
 
@@ -86,6 +97,11 @@ in brackets):
          MEDIA_ROOT
          TEMPLATE_DIRS
 
          MEDIA_ROOT
          TEMPLATE_DIRS
 
+        If you wish to enable the XML-RPC interface, add the following to
+        your local_settings.py file:
+
+         ENABLE_XMLRPC = True
+
        Then, get patchwork to create its tables in your configured database:
 
         cd apps/
        Then, get patchwork to create its tables in your configured database:
 
         cd apps/