A search interface for the Performing Patents Otherwise publication as part of the Politics of Patents case study (part of Copim WP6): this parses data from the archive of RTF files and provides additional data from the European Patent Office OPS API. https://patents.copim.ac.uk
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

188 line
6.8KB

  1. # @name: solr.py
  2. # @creation_date: 2022-09-07
  3. # @license: The MIT License <https://opensource.org/licenses/MIT>
  4. # @author: Simon Bowie <simon.bowie.19@gmail.com>
  5. # @purpose: Performs Solr functions
  6. # @acknowledgements:
  7. import os
  8. import requests
  9. import re
  10. import urllib
  11. import random
  12. import pycountry
  13. from . import ops
  14. # get config variables from OS environment variables: set in env file passed through Docker Compose
  15. solr_hostname = os.environ.get('SOLR_HOSTNAME')
  16. solr_port = os.environ.get('SOLR_PORT')
  17. def solr_search(core, sort, search=None, id=None):
  18. # Assemble a query string to send to Solr. This uses the Solr hostname from config.env. Solr's query syntax can be found at many sites including https://lucene.apache.org/solr/guide/6_6/the-standard-query-parser.html
  19. if id is not None:
  20. solrurl = 'http://' + solr_hostname + ':' + solr_port + '/solr/' + core + '/select?q.op=OR&q=id%3A"' + id + '"&wt=json'
  21. else:
  22. if (sort == 'relevance'):
  23. solrurl = 'http://' + solr_hostname + ':' + solr_port + '/solr/' + core + '/select?q.op=OR&q=content%3A' + urllib.parse.quote_plus(search) + '&wt=json'
  24. else:
  25. solrurl = 'http://' + solr_hostname + ':' + solr_port + '/solr/' + core + '/select?q.op=OR&q=content%3A' + urllib.parse.quote_plus(search) + '&wt=json&sort=' + sort
  26. # get result
  27. request = requests.get(solrurl)
  28. # turn the API response into useful Json
  29. json = request.json()
  30. num_found = json['response']['numFound']
  31. if (num_found == 0):
  32. output = 'no results found'
  33. else:
  34. output = []
  35. for result in json['response']['docs']:
  36. # set ID variable
  37. id = result['id']
  38. # set content variable
  39. content = result['content']
  40. # parse result
  41. result_output = parse_result(id, content)
  42. output.append(result_output)
  43. return output, num_found
  44. def parse_result(id, input):
  45. output = {}
  46. output['id'] = id
  47. # set document reference number (used for OPS API)
  48. doc_ref = re.search('=D\s(([^\s]*)\s([^\s]*)\s([^\s]*))', input)
  49. if doc_ref is None:
  50. doc_ref = re.search('=D&locale=en_EP\s(([^\s]*)\s([^\s]*)\s([^\s]*))', input)
  51. if doc_ref is None:
  52. output['doc_ref'] = ""
  53. else:
  54. output['doc_ref'] = doc_ref.group(1).replace(" ","")
  55. else:
  56. output['doc_ref'] = doc_ref.group(1).replace(" ","")
  57. # search for the application ID in the content element and display it
  58. application_id = re.search('Application.*\n(.*)\n', input)
  59. output['application_id'] = application_id.group(1)
  60. # search for the EPO publication URL in the content element and display it
  61. epo_publication = re.search('Publication.*\n(.*)\n', input)
  62. output['epo_publication_url'] = epo_publication.group(1)
  63. # search for the IPC publication URL in the content element and display it
  64. ipc_publication = re.search('IPC.*\n(.*)\n', input)
  65. output['ipc_publication_url'] = ipc_publication.group(1)
  66. # search for the title in the content element and display it
  67. title = re.search('Title.*\n(.*)\n', input)
  68. if title is not None:
  69. output['title'] = title.group(1)
  70. # search for the abstract in the content element and display it
  71. abstract = re.search('Abstract.*\n(.*)\n', input)
  72. if abstract is None:
  73. abstract = re.search('\(.\) \\n\\n(.*)\\n', input)
  74. if abstract is not None:
  75. output['abstract'] = abstract.group(1);
  76. # search for the year in the content element and display it
  77. year = re.search('=D[^\s]*\s[^\s]*\s[^\s]*\s[^\s]*\s(\d{4})', input)
  78. if year is not None:
  79. output['year'] = year.group(1)
  80. # search for the country in the content element and display it
  81. country_code = re.search('FT=D[^\s]*\s(\w{2})', input)
  82. country = pycountry.countries.get(alpha_2=country_code.group(1))
  83. if country is None:
  84. country = pycountry.historic_countries.get(alpha_2=country_code.group(1))
  85. if country is not None:
  86. output['country'] = country
  87. return output
  88. def get_random_record(core):
  89. rand = str(random.randint(0, 9999999))
  90. # Assemble a query string to send to Solr. This uses the Solr hostname from config.env. Solr's query syntax can be found at many sites including https://lucene.apache.org/solr/guide/6_6/the-standard-query-parser.html
  91. solrurl = 'http://' + solr_hostname + ':' + solr_port + '/solr/' + core + '/select?q.op=OR&q=*%3A*&wt=json&sort=random_' + rand + '%20asc&rows=1'
  92. # get result
  93. request = requests.get(solrurl)
  94. # turn the API response into useful Json
  95. json = request.json()
  96. if (json['response']['numFound'] == 0):
  97. output = 'no results found'
  98. else:
  99. output = []
  100. for result in json['response']['docs']:
  101. # set ID variables
  102. id = result['id']
  103. # set content variable
  104. content = result['content']
  105. # parse result
  106. result_output = parse_result(id, content)
  107. output.append(result_output)
  108. return output
  109. def get_ten_random_elements(field):
  110. core = 'all'
  111. output = []
  112. i = 0
  113. while i <= 9:
  114. results = get_random_record(core)
  115. for result in results:
  116. if field in result:
  117. dict = {'id': result['id'], field: result[field]}
  118. output.append(dict)
  119. i += 1
  120. return output
  121. def get_ten_random_images():
  122. core = 'all'
  123. output = []
  124. i = 0
  125. while i <= 9:
  126. results = get_random_record(core)
  127. for result in results:
  128. if ops.get_images(result['doc_ref']):
  129. image = ops.get_images(result['doc_ref'])
  130. result.update(image)
  131. output.append(result)
  132. i += 1
  133. return output
  134. def get_total_number(core):
  135. # Assemble a query string to send to Solr. This uses the Solr hostname from config.env. Solr's query syntax can be found at many sites including https://lucene.apache.org/solr/guide/6_6/the-standard-query-parser.html
  136. solrurl = 'http://' + solr_hostname + ':' + solr_port + '/solr/' + core + '/select?q.op=OR&q=*:*&wt=json'
  137. # get result
  138. request = requests.get(solrurl)
  139. # turn the API response into useful Json
  140. json = request.json()
  141. num_found = json['response']['numFound']
  142. return num_found
  143. def get_term_data(field, core):
  144. # Assemble a query string to send to Solr. This uses the Solr hostname from config.env. Solr's query syntax can be found at many sites including https://lucene.apache.org/solr/guide/6_6/the-standard-query-parser.html
  145. solrurl = 'http://' + solr_hostname + ':' + solr_port + '/solr/' + core + '/terms?terms.fl=' + field + '&wt=json&terms.limit=1000'
  146. # get result
  147. request = requests.get(solrurl)
  148. # turn the API response into useful Json
  149. json = request.json()
  150. output = json['terms'][field]
  151. return output