Commit 3b8a61c6 authored by Jeremy Kerr's avatar Jeremy Kerr
Browse files

Add patch tag infrastructure



This change add patch 'tags', eg 'Acked-by' / 'Reviewed-by', etc., to
patchwork.

Tag parsing is implemented in the patch parser's extract_tags function,
which returns a Counter object of the tags in a comment. These are
stored in the PatchTag (keyed to Tag) objects associated with each
patch.

We need to ensure that the main patch lists do not cause per-patch
queries on the Patch.tags ManyToManyField (this would result in ~500
queries per page), so we introduce a new QuerySet (and Manager) for
Patch, adding a with_tag_counts() method to populate the tag counts in a
single query.

As users may be migrating from previous patchwork versions (ie, with no
tag counts in the database), we add a 'retag' management command.
Signed-off-by: default avatarJeremy Kerr <jk@ozlabs.org>
parent daa3ae42
......@@ -148,7 +148,12 @@ in brackets):
PYTHONPATH=lib/python ./manage.py collectstatic
and add privileges for your mail and web users. This is only needed if
If you'd like to use the default tag set (Acked-by, Reviewed-by and
Tested-by), then load these default tags:
PYTHONPATH=lib/python ./manage.py loaddata default_tags
Finally, add privileges for your mail and web users. This is only needed if
you use the ident-based approach. If you use password-based database
authentication, you can skip this step.
......
......@@ -23,13 +23,17 @@ GRANT SELECT, UPDATE, INSERT, DELETE ON patchwork_bundlepatch TO 'www-data'@loca
GRANT SELECT, UPDATE, INSERT, DELETE ON patchwork_patch TO 'www-data'@localhost;
GRANT SELECT, UPDATE, INSERT, DELETE ON patchwork_emailoptout TO 'www-data'@localhost;
GRANT SELECT, UPDATE, INSERT, DELETE ON patchwork_patchchangenotification TO 'www-data'@localhost;
GRANT SELECT, UPDATE, INSERT, DELETE ON patchwork_tag TO 'www-data'@localhost;
GRANT SELECT, UPDATE, INSERT, DELETE ON patchwork_patchtag TO 'www-data'@localhost;
-- allow the mail user (in this case, 'nobody') to add patches
GRANT INSERT, SELECT ON patchwork_patch TO 'nobody'@localhost;
GRANT INSERT, SELECT ON patchwork_comment TO 'nobody'@localhost;
GRANT INSERT, SELECT ON patchwork_person TO 'nobody'@localhost;
GRANT INSERT, SELECT, UPDATE, DELETE ON patchwork_patchtag TO 'nobody'@localhost;
GRANT SELECT ON patchwork_project TO 'nobody'@localhost;
GRANT SELECT ON patchwork_state TO 'nobody'@localhost;
GRANT SELECT ON patchwork_tag TO 'nobody'@localhost;
COMMIT;
......@@ -23,7 +23,9 @@ GRANT SELECT, UPDATE, INSERT, DELETE ON
patchwork_bundlepatch,
patchwork_patch,
patchwork_emailoptout,
patchwork_patchchangenotification
patchwork_patchchangenotification,
patchwork_tag,
patchwork_patchtag
TO "www-data";
GRANT SELECT, UPDATE ON
auth_group_id_seq,
......@@ -44,7 +46,9 @@ GRANT SELECT, UPDATE ON
patchwork_state_id_seq,
patchwork_emailconfirmation_id_seq,
patchwork_userprofile_id_seq,
patchwork_userprofile_maintainer_projects_id_seq
patchwork_userprofile_maintainer_projects_id_seq,
patchwork_tag_id_seq,
patchwork_patchtag_id_seq
TO "www-data";
-- allow the mail user (in this case, 'nobody') to add patches
......@@ -53,14 +57,19 @@ GRANT INSERT, SELECT ON
patchwork_comment,
patchwork_person
TO "nobody";
GRANT INSERT, SELECT, UPDATE, DELETE ON
patchwork_patchtag
TO "nobody";
GRANT SELECT ON
patchwork_project,
patchwork_state
patchwork_state,
patchwork_tag
TO "nobody";
GRANT UPDATE, SELECT ON
patchwork_patch_id_seq,
patchwork_person_id_seq,
patchwork_comment_id_seq
patchwork_comment_id_seq,
patchwork_patchtag_id_seq
TO "nobody";
COMMIT;
......
BEGIN;
ALTER TABLE patchwork_project ADD COLUMN use_tags boolean default true;
CREATE TABLE "patchwork_tag" (
"id" serial NOT NULL PRIMARY KEY,
"name" varchar(20) NOT NULL,
"pattern" varchar(50) NOT NULL,
"abbrev" varchar(2) NOT NULL UNIQUE
);
CREATE TABLE "patchwork_patchtag" (
"id" serial NOT NULL PRIMARY KEY,
"patch_id" integer NOT NULL,
"tag_id" integer NOT NULL REFERENCES "patchwork_tag" ("id"),
"count" integer NOT NULL,
UNIQUE ("patch_id", "tag_id")
);
COMMIT;
from django.contrib import admin
from patchwork.models import Project, Person, UserProfile, State, Patch, \
Comment, Bundle
Comment, Bundle, Tag
class ProjectAdmin(admin.ModelAdmin):
list_display = ('name', 'linkname','listid', 'listemail')
......@@ -48,3 +48,8 @@ class BundleAdmin(admin.ModelAdmin):
list_filter = ('public', 'project')
search_fields = ('name', 'owner')
admin.site.register(Bundle, BundleAdmin)
class TagAdmin(admin.ModelAdmin):
list_display = ('name',)
admin.site.register(Tag, TagAdmin)
<?xml version="1.0" encoding="utf-8"?>
<django-objects version="1.0">
<object pk="1" model="patchwork.tag">
<field type="CharField" name="name">Acked-by</field>
<field type="CharField" name="pattern">^Acked-by:</field>
<field type="CharField" name="abbrev">A</field>
</object>
<object pk="2" model="patchwork.tag">
<field type="CharField" name="name">Reviewed-by</field>
<field type="CharField" name="pattern">^Reviewed-by:</field>
<field type="CharField" name="abbrev">R</field>
</object>
<object pk="3" model="patchwork.tag">
<field type="CharField" name="name">Tested-by</field>
<field type="CharField" name="pattern">^Tested-by:</field>
<field type="CharField" name="abbrev">T</field>
</object>
</django-objects>
\ No newline at end of file
from django.core.management.base import BaseCommand, CommandError
from patchwork.models import Patch
import sys
class Command(BaseCommand):
help = 'Update the tag (Ack/Review/Test) counts on existing patches'
args = '[<patch_id>...]'
def handle(self, *args, **options):
qs = Patch.objects
if args:
qs = qs.filter(id__in = args)
count = qs.count()
i = 0
for patch in qs.iterator():
patch.refresh_tag_counts()
i += 1
if (i % 10) == 0 or i == count:
sys.stdout.write('%06d/%06d\r' % (i, count))
sys.stdout.flush()
sys.stderr.write('\ndone\n')
......@@ -22,11 +22,13 @@ from django.contrib.auth.models import User
from django.core.urlresolvers import reverse
from django.contrib.sites.models import Site
from django.conf import settings
from patchwork.parser import hash_patch
from django.utils.functional import cached_property
from patchwork.parser import hash_patch, extract_tags
import re
import datetime, time
import random
from collections import Counter, OrderedDict
class Person(models.Model):
email = models.CharField(max_length=255, unique = True)
......@@ -56,6 +58,7 @@ class Project(models.Model):
scm_url = models.CharField(max_length=2000, blank=True)
webscm_url = models.CharField(max_length=2000, blank=True)
send_notifications = models.BooleanField(default=False)
use_tags = models.BooleanField(default=True)
def __unicode__(self):
return self.name
......@@ -65,6 +68,12 @@ class Project(models.Model):
return False
return self in user.profile.maintainer_projects.all()
@cached_property
def tags(self):
if not self.use_tags:
return []
return list(Tag.objects.all())
class Meta:
ordering = ['linkname']
......@@ -165,9 +174,68 @@ class HashField(models.CharField):
def db_type(self, connection=None):
return 'char(%d)' % self.n_bytes
class Tag(models.Model):
name = models.CharField(max_length=20)
pattern = models.CharField(max_length=50,
help_text='A simple regex to match the tag in the content of '
'a message. Will be used with MULTILINE and IGNORECASE '
'flags. eg. ^Acked-by:')
abbrev = models.CharField(max_length=2, unique=True,
help_text='Short (one-or-two letter) abbreviation for the tag, '
'used in table column headers')
def __unicode__(self):
return self.name
@property
def attr_name(self):
return 'tag_%d_count' % self.id
class Meta:
ordering = ['abbrev']
class PatchTag(models.Model):
patch = models.ForeignKey('Patch')
tag = models.ForeignKey('Tag')
count = models.IntegerField(default=1)
class Meta:
unique_together = [('patch', 'tag')]
def get_default_initial_patch_state():
return State.objects.get(ordering=0)
class PatchQuerySet(models.query.QuerySet):
def with_tag_counts(self, project):
if not project.use_tags:
return self
# We need the project's use_tags field loaded for Project.tags().
# Using prefetch_related means we'll share the one instance of
# Project, and share the project.tags cache between all patch.project
# references.
qs = self.prefetch_related('project')
select = OrderedDict()
select_params = []
for tag in project.tags:
select[tag.attr_name] = ("coalesce("
"(SELECT count FROM patchwork_patchtag "
"WHERE patchwork_patchtag.patch_id=patchwork_patch.id "
"AND patchwork_patchtag.tag_id=%s), 0)")
select_params.append(tag.id)
return qs.extra(select=select, select_params=select_params)
class PatchManager(models.Manager):
use_for_related_fields = True
def get_queryset(self):
return PatchQuerySet(self.model, using=self.db)
def with_tag_counts(self, project):
return self.get_queryset().with_tag_counts(project)
class Patch(models.Model):
project = models.ForeignKey(Project)
msgid = models.CharField(max_length=255)
......@@ -182,6 +250,9 @@ class Patch(models.Model):
pull_url = models.CharField(max_length=255, null = True, blank = True)
commit_ref = models.CharField(max_length=255, null = True, blank = True)
hash = HashField(null = True, blank = True)
tags = models.ManyToManyField(Tag, through=PatchTag)
objects = PatchManager()
def __unicode__(self):
return self.name
......@@ -189,6 +260,24 @@ class Patch(models.Model):
def comments(self):
return Comment.objects.filter(patch = self)
def _set_tag(self, tag, count):
if count == 0:
self.patchtag_set.filter(tag=tag).delete()
return
(patchtag, _) = PatchTag.objects.get_or_create(patch=self, tag=tag)
if patchtag.count != count:
patchtag.count = count
patchtag.save()
def refresh_tag_counts(self):
tags = self.project.tags
counter = Counter()
for comment in self.comment_set.all():
counter = counter + extract_tags(comment.content, tags)
for tag in tags:
self._set_tag(tag, counter[tag])
def save(self):
try:
s = self.state
......@@ -239,6 +328,14 @@ class Comment(models.Model):
return ''.join([ match.group(0) + '\n' for match in
self.response_re.finditer(self.content)])
def save(self, *args, **kwargs):
super(Comment, self).save(*args, **kwargs)
self.patch.refresh_tag_counts()
def delete(self, *args, **kwargs):
super(Comment, self).delete(*args, **kwargs)
self.patch.refresh_tag_counts()
class Meta:
ordering = ['date']
unique_together = [('msgid', 'patch')]
......
......@@ -21,6 +21,7 @@
import re
from collections import Counter
try:
import hashlib
......@@ -234,6 +235,14 @@ def hash_patch(str):
return hash
def extract_tags(content, tags):
counts = Counter()
for tag in tags:
regex = re.compile(tag.pattern, re.MULTILINE | re.IGNORECASE)
counts[tag] = len(regex.findall(content))
return counts
def main(args):
from optparse import OptionParser
......
{% load person %}
{% load listurl %}
{% load patch %}
{% load static %}
{% include "patchwork/pagination.html" %}
......@@ -68,6 +69,12 @@
{% endifequal %}
</th>
<th>
<span
title="{% for tag in project.tags %}{{tag.name}}{% if not forloop.last %} / {% endif %}{% endfor %}"
>{% for tag in project.tags %}{{tag.abbrev}}{% if not forloop.last %}/{% endif %}{% endfor %}</span>
</th>
<th>
{% ifequal order.name "date" %}
<a class="colactive"
......@@ -166,6 +173,7 @@
{% endif %}
<td><a href="{% url 'patchwork.views.patch.patch' patch_id=patch.id %}"
>{{ patch.name|default:"[no subject]" }}</a></td>
<td style="white-space: nowrap;">{{ patch|patch_tags }}</td>
<td>{{ patch.date|date:"Y-m-d" }}</td>
<td>{{ patch.submitter|personify:project }}</td>
<td>{{ patch.delegate.username }}</td>
......@@ -259,7 +267,7 @@
{% else %}
<tr>
<td colspan="5">No patches to display</td>
<td colspan="6">No patches to display</td>
</tr>
{% endif %}
......
......@@ -18,6 +18,7 @@
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
from django import template
from django.utils.safestring import mark_safe
import re
register = template.Library()
......@@ -63,3 +64,15 @@ class EditablePatchNode(template.Node):
return self.nodelist_false.render(context)
return self.nodelist_true.render(context)
@register.filter(name='patch_tags')
def patch_tags(patch):
counts = []
titles = []
for tag in patch.project.tags:
count = getattr(patch, tag.attr_name)
titles.append('%d %s' % (count, tag.name))
counts.append(str(count))
return mark_safe('<span title="%s">%s</span>' % (
' / '.join(titles),
' '.join(counts)))
......@@ -18,6 +18,7 @@
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
from patchwork.tests.test_patchparser import *
from patchwork.tests.test_tags import *
from patchwork.tests.test_encodings import *
from patchwork.tests.test_bundles import *
from patchwork.tests.test_mboxviews import *
......
......@@ -552,3 +552,30 @@ class InitialPatchStateTest(TestCase):
def tearDown(self):
self.p1.delete()
self.user.delete()
class ParseInitialTagsTest(PatchTest):
patch_filename = '0001-add-line.patch'
test_comment = ('test comment\n\n' +
'Tested-by: Test User <test@example.com>\n' +
'Reviewed-by: Test User <test@example.com>\n')
fixtures = ['default_tags']
def setUp(self):
project = defaults.project
project.listid = 'test.example.com'
project.save()
self.orig_patch = read_patch(self.patch_filename)
email = create_email(self.test_comment + '\n' + self.orig_patch,
project = project)
email['Message-Id'] = '<1@example.com>'
parse_mail(email)
def testTags(self):
self.assertEquals(Patch.objects.count(), 1)
patch = Patch.objects.all()[0]
self.assertEquals(patch.patchtag_set.filter(
tag__name='Acked-by').count(), 0)
self.assertEquals(patch.patchtag_set.get(
tag__name='Reviewed-by').count, 1)
self.assertEquals(patch.patchtag_set.get(
tag__name='Tested-by').count, 1)
# Patchwork - automated patch tracking system
# Copyright (C) 2014 Jeremy Kerr <jk@ozlabs.org>
#
# This file is part of the Patchwork package.
#
# Patchwork is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# Patchwork is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Patchwork; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import unittest
import datetime
from django.test import TestCase, TransactionTestCase
from patchwork.models import Project, Patch, Comment, Tag, PatchTag
from patchwork.tests.utils import defaults
from patchwork.parser import extract_tags
from django.conf import settings
from django.db import connection
class ExtractTagsTest(TestCase):
email = 'test@exmaple.com'
name_email = 'test name <' + email + '>'
fixtures = ['default_tags']
def assertTagsEqual(self, str, acks, reviews, tests):
counts = extract_tags(str, Tag.objects.all())
self.assertEquals((acks, reviews, tests),
(counts[Tag.objects.get(name='Acked-by')],
counts[Tag.objects.get(name='Reviewed-by')],
counts[Tag.objects.get(name='Tested-by')]))
def testEmpty(self):
self.assertTagsEqual("", 0, 0, 0)
def testNoTag(self):
self.assertTagsEqual("foo", 0, 0, 0)
def testAck(self):
self.assertTagsEqual("Acked-by: %s" % self.name_email, 1, 0, 0)
def testAckEmailOnly(self):
self.assertTagsEqual("Acked-by: %s" % self.email, 1, 0, 0)
def testReviewed(self):
self.assertTagsEqual("Reviewed-by: %s" % self.name_email, 0, 1, 0)
def testTested(self):
self.assertTagsEqual("Tested-by: %s" % self.name_email, 0, 0, 1)
def testAckAfterNewline(self):
self.assertTagsEqual("\nAcked-by: %s" % self.name_email, 1, 0, 0)
def testMultipleAcks(self):
str = "Acked-by: %s\nAcked-by: %s\n" % ((self.name_email,) * 2)
self.assertTagsEqual(str, 2, 0, 0)
def testMultipleTypes(self):
str = "Acked-by: %s\nAcked-by: %s\nReviewed-by: %s\n" % (
(self.name_email,) * 3)
self.assertTagsEqual(str, 2, 1, 0)
def testLower(self):
self.assertTagsEqual("acked-by: %s" % self.name_email, 1, 0, 0)
def testUpper(self):
self.assertTagsEqual("ACKED-BY: %s" % self.name_email, 1, 0, 0)
def testAckInReply(self):
self.assertTagsEqual("> Acked-by: %s\n" % self.name_email, 0, 0, 0)
class PatchTagsTest(TransactionTestCase):
ACK = 1
REVIEW = 2
TEST = 3
fixtures = ['default_tags']
def assertTagsEqual(self, patch, acks, reviews, tests):
patch = Patch.objects.get(pk=patch.pk)
def count(name):
try:
return patch.patchtag_set.get(tag__name=name).count
except PatchTag.DoesNotExist:
return 0
counts = (
count(name='Acked-by'),
count(name='Reviewed-by'),
count(name='Tested-by'),
)
self.assertEqual(counts, (acks, reviews, tests))
def create_tag(self, tagtype = None):
tags = {
self.ACK: 'Acked',
self.REVIEW: 'Reviewed',
self.TEST: 'Tested'
}
if tagtype not in tags:
return ''
return '%s-by: %s\n' % (tags[tagtype], self.tagger)
def create_tag_comment(self, patch, tagtype = None):
comment = Comment(patch=patch, msgid=str(datetime.datetime.now()),
submitter=defaults.patch_author_person,
content=self.create_tag(tagtype))
comment.save()
return comment
def setUp(self):
settings.DEBUG = True
project = Project(linkname='test-project', name='Test Project',
use_tags=True)
project.save()
defaults.patch_author_person.save()
self.patch = Patch(project=project,
msgid='x', name=defaults.patch_name,
submitter=defaults.patch_author_person,
content='')
self.patch.save()
self.tagger = 'Test Tagger <tagger@example.com>'
def tearDown(self):
self.patch.delete()
def testNoComments(self):
self.assertTagsEqual(self.patch, 0, 0, 0)
def testNoTagComment(self):
self.create_tag_comment(self.patch, None)
self.assertTagsEqual(self.patch, 0, 0, 0)
def testSingleComment(self):
self.create_tag_comment(self.patch, self.ACK)
self.assertTagsEqual(self.patch, 1, 0, 0)
def testMultipleComments(self):
self.create_tag_comment(self.patch, self.ACK)
self.create_tag_comment(self.patch, self.ACK)
self.assertTagsEqual(self.patch, 2, 0, 0)
def testMultipleCommentTypes(self):
self.create_tag_comment(self.patch, self.ACK)
self.create_tag_comment(self.patch, self.REVIEW)
self.create_tag_comment(self.patch, self.TEST)
self.assertTagsEqual(self.patch, 1, 1, 1)
def testCommentAdd(self):
self.create_tag_comment(self.patch, self.ACK)
self.assertTagsEqual(self.patch, 1, 0, 0)
self.create_tag_comment(self.patch, self.ACK)
self.assertTagsEqual(self.patch, 2, 0, 0)