A search interface for the Performing Patents Otherwise publication as part of the Politics of Patents case study (part of Copim WP6): this parses data from the archive of RTF files and provides additional data from the European Patent Office OPS API. https://patents.copim.ac.uk
Ви не можете вибрати більше 25 тем Теми мають розпочинатися з літери або цифри, можуть містити дефіси (-) і не повинні перевищувати 35 символів.

192 lines
6.9KB

  1. # @name: solr.py
  2. # @creation_date: 2022-09-07
  3. # @license: The MIT License <https://opensource.org/licenses/MIT>
  4. # @author: Simon Bowie <simon.bowie.19@gmail.com>
  5. # @purpose: Performs Solr functions
  6. # @acknowledgements:
  7. import os
  8. import requests
  9. import re
  10. import urllib
  11. import random
  12. import pycountry
  13. from . import ops
  14. # get config variables from OS environment variables: set in env file passed through Docker Compose
  15. solr_hostname = os.environ.get('SOLR_HOSTNAME')
  16. solr_port = os.environ.get('SOLR_PORT')
  17. def solr_search(core, sort, search=None, id=None):
  18. # Assemble a query string to send to Solr. This uses the Solr hostname from config.env. Solr's query syntax can be found at many sites including https://lucene.apache.org/solr/guide/6_6/the-standard-query-parser.html
  19. if id is not None:
  20. solrurl = 'http://' + solr_hostname + ':' + solr_port + '/solr/' + core + '/select?q.op=OR&q=id%3A"' + id + '"&wt=json'
  21. else:
  22. if (sort == 'relevance'):
  23. solrurl = 'http://' + solr_hostname + ':' + solr_port + '/solr/' + core + '/select?q.op=OR&q=content%3A' + urllib.parse.quote_plus(search) + '&wt=json'
  24. else:
  25. solrurl = 'http://' + solr_hostname + ':' + solr_port + '/solr/' + core + '/select?q.op=OR&q=content%3A' + urllib.parse.quote_plus(search) + '&wt=json&sort=' + sort
  26. # get result
  27. request = requests.get(solrurl)
  28. # turn the API response into useful Json
  29. json = request.json()
  30. num_found = json['response']['numFound']
  31. if (num_found == 0):
  32. output = 'no results found'
  33. else:
  34. output = []
  35. for result in json['response']['docs']:
  36. # set ID variable
  37. id = result['id']
  38. # set content variable
  39. content = result['content']
  40. # parse result
  41. result_output = parse_result(id, content)
  42. output.append(result_output)
  43. return output, num_found
  44. def parse_result(id, input):
  45. output = {}
  46. output['id'] = id
  47. # set document reference number (used for OPS API)
  48. doc_ref = re.search('=D\s(([^\s]*)\s([^\s]*)\s([^\s]*))', input)
  49. if doc_ref is None:
  50. doc_ref = re.search('=D&locale=en_EP\s(([^\s]*)\s([^\s]*)\s([^\s]*))', input)
  51. if doc_ref is None:
  52. output['doc_ref'] = ""
  53. else:
  54. output['doc_ref'] = doc_ref.group(1).replace(" ","")
  55. else:
  56. output['doc_ref'] = doc_ref.group(1).replace(" ","")
  57. # search for the application ID in the content element and display it
  58. application_id = re.search('Application.*\n(.*)\n', input)
  59. output['application_id'] = application_id.group(1)
  60. # search for the EPO publication URL in the content element and display it
  61. epo_publication = re.search('Publication.*\n(.*)\n', input)
  62. output['epo_publication_url'] = epo_publication.group(1)
  63. # search for the IPC publication URL in the content element and display it
  64. ipc_publication = re.search('IPC.*\n(.*)\n', input)
  65. output['ipc_publication_url'] = ipc_publication.group(1)
  66. # search for the title in the content element and display it
  67. title = re.search('Title.*\n(.*)\n', input)
  68. if title is not None:
  69. output['title'] = title.group(1)
  70. # search for the abstract in the content element and display it
  71. abstract = re.search('Abstract.*\n(.*)\n', input)
  72. if abstract is None:
  73. abstract = re.search('\(.\) \\n\\n(.*)\\n', input)
  74. if abstract is not None:
  75. output['abstract'] = abstract.group(1);
  76. # search for the year in the content element and display it
  77. year = re.search('=D[^\s]*\s[^\s]*\s[^\s]*\s[^\s]*\s(\d{4})', input)
  78. if year is not None:
  79. output['year'] = year.group(1)
  80. # search for the country in the content element and display it
  81. country_code = re.search('FT=D[^\s]*\s(\w{2})', input)
  82. country = pycountry.countries.get(alpha_2=country_code.group(1))
  83. if country is not None:
  84. output['country'] = country
  85. else:
  86. country = pycountry.historic_countries.get(alpha_2=country_code.group(1))
  87. if country is not None:
  88. output['country'] = country
  89. else:
  90. output['country'] = country_code.group(1)
  91. return output
  92. def get_random_record(core):
  93. rand = str(random.randint(0, 9999999))
  94. # Assemble a query string to send to Solr. This uses the Solr hostname from config.env. Solr's query syntax can be found at many sites including https://lucene.apache.org/solr/guide/6_6/the-standard-query-parser.html
  95. solrurl = 'http://' + solr_hostname + ':' + solr_port + '/solr/' + core + '/select?q.op=OR&q=*%3A*&wt=json&sort=random_' + rand + '%20asc&rows=1'
  96. # get result
  97. request = requests.get(solrurl)
  98. # turn the API response into useful Json
  99. json = request.json()
  100. if (json['response']['numFound'] == 0):
  101. output = 'no results found'
  102. else:
  103. output = []
  104. for result in json['response']['docs']:
  105. # set ID variables
  106. id = result['id']
  107. # set content variable
  108. content = result['content']
  109. # parse result
  110. result_output = parse_result(id, content)
  111. output.append(result_output)
  112. return output
  113. def get_ten_random_elements(field):
  114. core = 'all'
  115. output = []
  116. i = 0
  117. while i <= 9:
  118. results = get_random_record(core)
  119. for result in results:
  120. if field in result:
  121. dict = {'id': result['id'], field: result[field]}
  122. output.append(dict)
  123. i += 1
  124. return output
  125. def get_ten_random_images():
  126. core = 'all'
  127. output = []
  128. i = 0
  129. while i <= 9:
  130. results = get_random_record(core)
  131. for result in results:
  132. if ops.get_images(result['doc_ref']):
  133. image = ops.get_images(result['doc_ref'])
  134. result.update(image)
  135. output.append(result)
  136. i += 1
  137. return output
  138. def get_total_number(core):
  139. # Assemble a query string to send to Solr. This uses the Solr hostname from config.env. Solr's query syntax can be found at many sites including https://lucene.apache.org/solr/guide/6_6/the-standard-query-parser.html
  140. solrurl = 'http://' + solr_hostname + ':' + solr_port + '/solr/' + core + '/select?q.op=OR&q=*:*&wt=json'
  141. # get result
  142. request = requests.get(solrurl)
  143. # turn the API response into useful Json
  144. json = request.json()
  145. num_found = json['response']['numFound']
  146. return num_found
  147. def get_term_data(field, core):
  148. # Assemble a query string to send to Solr. This uses the Solr hostname from config.env. Solr's query syntax can be found at many sites including https://lucene.apache.org/solr/guide/6_6/the-standard-query-parser.html
  149. solrurl = 'http://' + solr_hostname + ':' + solr_port + '/solr/' + core + '/terms?terms.fl=' + field + '&wt=json&terms.limit=1000'
  150. # get result
  151. request = requests.get(solrurl)
  152. # turn the API response into useful Json
  153. json = request.json()
  154. output = json['terms'][field]
  155. return output