Commits

Anonymous committed df2fad4 Draft

Implemented methods from ISearchBackend for Solr

Comments (0)

Files changed (12)

bloodhound_solr/bhsolr/index.py

-import sunburnt
-from solr import Solr
-
-class SolrIndex(object):
-  def __init__(self, solr_instance):
-    self.solr_instance = solr_instance
-
-  def index(self, doc):
-    self.solr_instance.solr_interface.add(doc)
-    self.solr_instance.solr_interface.commit()
-
-  def query(self, query):
-    self.solr_instance.solr_interface.query(query)
-
-if __name__ == '__main__':
-
-  document = {"id":"0553573403",
-              "name": "Ticket 1"}
-
-  si = Solr("http://localhost:8983/solr/")
-  sindex = SolrIndex(si)
-  sindex.index(document)
-  sindex.query("Ticket")
-

bloodhound_solr/bhsolr/schemadoc/schema.xml

   <field name="_root_" type="string" indexed="true" stored="false"/>
 
   <!-- BH fields -->
+  <field name="id" type="string" indexed="true" stored="true" required="true" multiValued="false" />
   <field name="unique_id" type="string" indexed="true" stored="true" required="true" multiValued="false" />
-  <field name="id" type="string" indexed="true" stored="true" required="true" multiValued="false" />
   <field name="type" type="string" indexed="true" stored="true" required="true" multiValued="false"/>
-  <field name="product" type="string" indexed="true" stored="true" required="true" multiValued="false"/>
-  <field name="milestone" type="string" indexed="true" stored="true" required="true" multiValued="false"/>
-  <field name="time" type="date" indexed="true" stored="true" required="true" multiValued="false"/>
-  <field name="due" type="date" indexed="true" stored="true" required="true" multiValued="false"/>
-  <field name="completed" type="date" indexed="true" stored="true" required="true" multiValued="false"/>
-  <field name="author" type="string" indexed="true" stored="true" required="true" multiValued="false"/>
-  <field name="component" type="string" indexed="true" stored="true" required="true" multiValued="false"/>
-  <field name="status" type="string" indexed="true" stored="true" required="true" multiValued="false"/>
-  <field name="resolution" type="string" indexed="true" stored="true" required="true" multiValued="false"/>
-  <field name="keywords" type="string" indexed="true" stored="true" required="true" multiValued="false"/>
-  <field name="summary" type="text_general" indexed="true" stored="true" required="true" multiValued="false"/>
-  <field name="content" type="text_general" indexed="true" stored="true" required="true" multiValued="false"/>
-  <field name="changes" type="text_general" indexed="true" stored="true" required="true" multiValued="false"/>
-  <field name="owner" type="text_general" indexed="true" stored="true" required="true" multiValued="false"/>
-  <field name="repository" type="text_general" indexed="true" stored="true" required="true" multiValued="false"/>
-  <field name="revision" type="text_general" indexed="true" stored="true" required="true" multiValued="false"/>
-  <field name="message" type="text_general" indexed="true" stored="true" required="true" multiValued="false"/>
-  <field name="required_permission" type="string" indexed="true" stored="true" required="true" multiValued="false"/>
-  <field name="name" type="text_general" indexed="true" stored="true" required="true" multiValued="false"/>
-  <field name="query_suggestion_basket" type="text_general" indexed="true" stored="true" required="true" multiValued="false"/>
-  <field name="relations" type="lowercase" indexed="true" stored="true" required="true" multiValued="false"/>
-  <field/>
-
-
-  <field name="name" type="text_general" indexed="true" stored="true"/>
+  <field name="product" type="string" indexed="true" stored="true" required="false" multiValued="false"/>
+  <field name="milestone" type="string" indexed="true" stored="true" required="false" multiValued="false"/>
+  <field name="time" type="date" indexed="true" stored="true" required="false" multiValued="false"/>
+  <field name="due" type="date" indexed="true" stored="true" required="false" multiValued="false"/>
+  <field name="completed" type="date" indexed="true" stored="true" required="false" multiValued="false"/>
+  <field name="author" type="string" indexed="true" stored="true" required="false" multiValued="false"/>
+  <field name="component" type="string" indexed="true" stored="true" required="false" multiValued="false"/>
+  <field name="status" type="string" indexed="true" stored="true" required="false" multiValued="false"/>
+  <field name="resolution" type="string" indexed="true" stored="true" required="false" multiValued="false"/>
+  <field name="keywords" type="string" indexed="true" stored="true" required="false" multiValued="false"/>
+  <field name="summary" type="text_general" indexed="true" stored="true" required="false" multiValued="false"/>
+  <field name="content" type="text_general" indexed="true" stored="true" required="false" multiValued="false"/>
+  <field name="changes" type="text_general" indexed="true" stored="true" required="false" multiValued="false"/>
+  <field name="owner" type="text_general" indexed="true" stored="true" required="false" multiValued="false"/>
+  <field name="repository" type="text_general" indexed="true" stored="true" required="false" multiValued="false"/>
+  <field name="revision" type="text_general" indexed="true" stored="true" required="false" multiValued="false"/>
+  <field name="message" type="text_general" indexed="true" stored="true" required="false" multiValued="false"/>
+  <field name="required_permission" type="string" indexed="true" stored="true" required="false" multiValued="false"/>
+  <field name="name" type="text_general" indexed="true" stored="true" required="false" multiValued="false"/>
+  <field name="_stored_name" type="string" indexed="true" stored="true" required="false" multiValued="false"/>
+  <!-- <field name="query_suggestion_basket" type="text_general" indexed="true" stored="true" required="true" multiValued="false"/> -->
+  <!-- <field name="relations" type="lowercase" indexed="true" stored="true" required="true" multiValued="false"/> -->
 </fields>
 
 <uniqueKey>unique_id</uniqueKey>
 
-<copyField source="name" dest="text"/>
+<!-- <copyField source="name" dest="text"/> -->
 
 <types>
   <!-- Field type definitions -->
       <filter class="solr.LowerCaseFilterFactory"/>
     </analyzer>
   </fieldType>
+
   <fieldType name="date" class="solr.TrieDateField" precisionStep="0" positionIncrementGap="0"/>
+
   <fieldType name="lowercase" class="solr.TextField" positionIncrementGap="100">
-  <analyzer>
-    <tokenizer class="solr.KeywordTokenizerFactory"/>
-    <filter class="solr.LowerCaseFilterFactory" />
-  </analyzer>
-</fieldType>
+    <analyzer>
+      <tokenizer class="solr.KeywordTokenizerFactory"/>
+      <filter class="solr.LowerCaseFilterFactory" />
+    </analyzer>
+  </fieldType>
+</types>
 
 </schema>
 

bloodhound_solr/bhsolr/search_resources/__init__.py

Empty file added.

bloodhound_solr/bhsolr/search_resources/changeset_search.py

+from bhsearch.search_resources.base import BaseIndexer
+from trac.versioncontrol.api import RepositoryManager
+from bhsearch.search_resources.changeset_search import ChangesetIndexer
+
+class ChangesetSearchModel(BaseIndexer):
+
+  def get_entries_for_index(self):
+    repository_manager = RepositoryManager(self.env)
+    for repository in repository_manager.get_real_repositories():
+      rev = repository.oldest_rev
+      stop = repository.youngest_rev
+      while True:
+        changeset = repository.get_changeset(rev)
+        yield ChangesetIndexer(self.env).build_doc(changeset)
+        if rev == stop:
+          break
+        rev = repository.next_rev(rev)

bloodhound_solr/bhsolr/search_resources/milestone_search.py

+from bhsearch.search_resources.base import BaseIndexer
+from trac.ticket import Milestone
+from bhsearch.search_resources.milestone_search import MilestoneIndexer
+
+class MilestoneSearchModel(BaseIndexer):
+
+  def get_entries_for_index(self):
+    for milestone in Milestone.select(self.env, include_completed=True):
+      yield MilestoneIndexer(self.env).build_doc(milestone)

bloodhound_solr/bhsolr/search_resources/ticket_search.py

+from trac.ticket.model import Ticket
+from bhsearch.search_resources.ticket_search import TicketIndexer
+from trac.core import Component, implements, TracError
+from bhsearch.search_resources.base import BaseIndexer
+
+class TicketSearchModel(BaseIndexer):
+
+  def _fetch_tickets(self,  **kwargs):
+    for ticket_id in self._fetch_ids(**kwargs):
+      yield Ticket(self.env, ticket_id)
+
+  def _fetch_ids(self, **kwargs):
+    sql = "SELECT id FROM ticket"
+    args = []
+    conditions = []
+    for key, value in kwargs.iteritems():
+      args.append(value)
+      conditions.append(key + "=%s")
+    if conditions:
+      sql = sql + " WHERE " + " AND ".join(conditions)
+    for row in self.env.db_query(sql, args):
+      yield int(row[0])
+
+  def get_entries_for_index(self):
+    for ticket in self._fetch_tickets():
+      yield TicketIndexer(self.env).build_doc(ticket)
+

bloodhound_solr/bhsolr/search_resources/wiki_search.py

+from trac.wiki import WikiSystem, WikiPage
+from bhsearch.search_resources.wiki_search import WikiIndexer
+from bhsearch.search_resources.base import BaseIndexer
+
+class WikiSearchModel(BaseIndexer):
+
+  def get_entries_for_index(self):
+    page_names = WikiSystem(self.env).get_pages()
+    for page_name in page_names:
+      page = WikiPage(self.env, page_name)
+      yield WikiIndexer(self.env).build_doc(page)

bloodhound_solr/bhsolr/solr.py

 
 class Solr():
 
-  def __init__(self, solr_url, schema_file):
-  """ Creates a SolrInterface object with the solr server url and a custom schema
+  def __init__(self, solr_url):
+    """ Creates a SolrInterface object with the solr server url and a custom schema
       file"""
     self.solr_url = solr_url
-    self.solr_interface = SolrInterface(url=solr_url, schemadoc=schema_file)
+    self.solr_interface = SolrInterface(url=solr_url)

bloodhound_solr/bhsolr/solr_backend.py

+import pkg_resources
+
+from solr import Solr
+from trac.ticket.model import Ticket
+from trac.core import Component, implements, TracError
+from trac.ticket.api import TicketSystem
+from bhsearch.search_resources.ticket_search import TicketIndexer
+from datetime import datetime
+from trac.util.datefmt import utc
+
+UNIQUE_ID = "unique_id"
+
+class SolrModel(Component):
+  implements(ISearchBackend)
+
+  def __init__(self):
+    resource_filename = pkg_resources.resource_filename
+    path = resource_filename(__name__, "schemadoc")
+    # self.solr_interface = Solr("http://localhost:8983/solr/", path + '/schema.xml').solr_interface
+    self.solr_interface = Solr("http://localhost:8983/solr/").solr_interface
+
+
+  def add_doc(self, doc, operation_context=None):
+    self._reformat_doc(doc)
+    doc[UNIQUE_ID] = self._create_unique_id(doc.get("product", ''),
+                                            doc["type"],
+                                            doc["id"])
+    self.solr_interface.add(doc)
+    self.solr_interface.commit()
+
+
+  def delete_doc(product, doc_type, doc_id, operation_context=None):
+    unique_id = self._create_unique_id(product, doc_type, doc_id)
+    self.solr_interface.delete(unique_id)
+
+
+  def optimize():
+    self.solr_interface.optimize()
+
+
+  def query(self, query):
+    self.solr_instance.solr_interface.query(query).execute()
+
+
+  def _reformat_doc(self, doc):
+    for key, value in doc.items():
+      if key is None:
+        del doc[None]
+      elif value is None:
+        del doc[key]
+      elif isinstance(value, basestring) and value == "":
+        del doc[key]
+      else:
+        doc[key] = self._to_solr_format(value)
+
+
+  def _to_solr_format(self, value):
+    if isinstance(value, basestring):
+      value = unicode(value)
+    elif isinstance(value, datetime):
+      value = self._convert_date_to_tz_naive_utc(value)
+    return value
+
+
+  def _convert_date_to_tz_naive_utc(self, value):
+    if value.tzinfo:
+      utc_time = value.astimezone(utc)
+      value = utc_time.replace(tzinfo=None)
+    return value
+
+
+  def _create_unique_id(self, product, doc_type, doc_id):
+    if product:
+      return u"%s:%s:%s" % (product, doc_type, doc_id)
+    else:
+      return u"%s:%s" % (doc_type, doc_id)
+
+if __name__ == '__main__':
+  env = trac.env.Environment("/Users/antonia/Documents/Code/bloodhound/installer/bloodhound/environments/main")
+  db_connection = env.get_db_cnx()
+  cursor = db_connection.cursor()
+  a = cursor.execute("select * from ticket")
+  ticket = a.fetchall()[0]
+
+  # for result in si.query(name="Ticket").execute():
+    # print result

bloodhound_solr/bhsolr/templates/bh_solr_test.html

+<!DOCTYPE html
+    PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml"
+      xmlns:xi="http://www.w3.org/2001/XInclude"
+      xmlns:py="http://genshi.edgewall.org/"
+      xmlns:i18n="http://genshi.edgewall.org/i18n">
+      <link rel="stylesheet" href="${href.chrome('dashboard/css/bootstrap.css')}" type="text/css" />
+      <link rel="stylesheet" href="${href.chrome('dashboard/css/bootstrap-responsive.css')}" type="text/css" />
+
+  <head>
+    <title>Test</title>
+  </head>
+</html>

bloodhound_solr/bhsolr/web_ui.py

+import re
+import os
+import pkg_resources
+
+from trac.web.main import IRequestHandler
+from trac.core import Component, TracError, implements
+from trac.ticket.model import Ticket
+from trac.web.chrome import ITemplateProvider
+from bhsolr.search_resources.ticket_search import TicketSearchModel
+from bhsolr.search_resources.milestone_search import MilestoneSearchModel
+from bhsolr.search_resources.changeset_search import ChangesetSearchModel
+from bhsolr.search_resources.wiki_search import WikiSearchModel
+from bhsolr.solr_backend import SolrModel
+
+class BloodhoundSolrSearchModule(Component):
+    implements(IRequestHandler, ITemplateProvider)
+
+    def match_request(self, req):
+      if re.match(r'/solr$', req.path_info):
+        return True
+
+    def process_request(self, req):
+      # changeset_doc = next(ChangesetSearchModel(self.env).get_entries_for_index())
+      milestone_doc = next(MilestoneSearchModel(self.env).get_entries_for_index())
+      ticket_doc = next(TicketSearchModel(self.env).get_entries_for_index())
+      wiki_doc = next(WikiSearchModel(self.env).get_entries_for_index())
+
+      SolrModel(self.env).addDoc(ticket_doc)
+      SolrModel(self.env).addDoc(milestone_doc)
+      SolrModel(self.env).addDoc(wiki_doc)
+
+      data = {}
+      return 'bh_solr_test.html', data, None
+
+    def get_templates_dirs(self):
+        resource_filename = pkg_resources.resource_filename
+        return [resource_filename('bhsolr', 'templates')]
+
+    def get_htdocs_dirs(self):
+        resource_filename = pkg_resources.resource_filename
+        return [('solr', resource_filename('bhsolr', 'htdocs'))]
+

bloodhound_solr/setup.py

-"""setup for embeddable objects plugin"""
-from setuptools import setup
+from setuptools import setup, find_packages
+
+PKG_INFO = {'bhsolr': ['htdocs/*.*', 'templates/*', 'schemadoc/*.xml'],
+            'bhsolr.search_resources' : [],
+            }
+
+
+
+ENTRY_POINTS = {
+          'trac.plugins': [
+          'bhsolr.web_ui = bhsolr.web_ui',
+          'bhsolr.api = bhsolr.api',
+          'bhsolr.solr = bhsolr.solr',
+          'bhsolr.solr_backend = bhsolr.solr_backend',
+          'bhsolr.search_resources.ticket_search = bhsolr.search_resources.ticket_search',
+          'bhsolr.search_resources.milestone_search = bhsolr.search_resources.milestone_search',
+          'bhsolr.search_resources.changeset_search = bhsolr.search_resources.changeset_search',
+          'bhsolr.search_resources.wiki_search = bhsolr.search_resources.wiki_search'
+      ],}
 
 setup(
-    name = 'BloodhoundSolrPlugin',
-    version = '0.1',
-    description = "Apache Solr support for Apache(TM) Bloodhound.",
-    author = "Apache Bloodhound",
-    license = "Apache License v2",
-    url = "http://bloodhound.apache.org/",
-    packages = ['bhsolr',],
-    package_data = {'bhsolr' : []},
-    entry_points = {'trac.plugins': ['bhsolr.index = bhsolr.index'],},
-    test_suite='bhsorl.tests.test_suite',
+  name = 'BloodhoundSolrPlugin',
+  version = '0.1',
+  description = "Apache Solr support for Apache(TM) Bloodhound.",
+  author = "Apache Bloodhound",
+  license = "Apache License v2",
+  url = "http://bloodhound.apache.org/",
+  # package_dir = PKG_INFO,
+  packages = find_packages(),
+  package_data = PKG_INFO,
+  include_package_data=True,
+  entry_points = ENTRY_POINTS,
+  test_suite='bhsolr.tests.test_suite',
 )
-
-