Commit 1f1ce72f authored by Aron Gergely's avatar Aron Gergely

Implemented full text search on tsvector table

parent 7ae1a33b
......@@ -28,3 +28,6 @@ python3.5 src/manager.py import_shelters_documents data/shelters/documents/
# Import of the translations
#python3.5 src/manager.py import_translation data/translations/sheltersDataTraduction_FR_rev_ED.csv fr
# Create database triggers for materialized view table
python3.5 src/manager.py create_db_triggers
......@@ -118,5 +118,12 @@ def import_shelters_documents(folder):
with app.app_context():
scripts.import_shelters_documents(folder)
@manager.command
def create_db_triggers():
"Creates triggers and trigger functions for tables"
print("Creating database triggers ...")
with app.app_context():
scripts.create_db_triggers()
if __name__ == '__main__':
manager.run()
......@@ -8,3 +8,4 @@ from .init_page import init_page
from .import_translation import import_translation
from .import_shelters_pictures import import_shelters_pictures
from .import_shelters_documents import import_shelters_documents
from .create_db_triggers import create_db_triggers
from bootstrap import db
def create_db_triggers():
db.engine.execute(
"""
CREATE OR REPLACE FUNCTION refresh_materialized_view() RETURNS trigger AS
$$
BEGIN
REFRESH MATERIALIZED VIEW tsvector;
RETURN NULL;
END;
$$
LANGUAGE plpgsql;
CREATE TRIGGER refresh_shelter AFTER TRUNCATE OR INSERT OR UPDATE OR DELETE
ON shelter FOR EACH STATEMENT
EXECUTE PROCEDURE refresh_materialized_view();
CREATE TRIGGER refresh_property AFTER TRUNCATE OR INSERT OR UPDATE OR DELETE
ON property FOR EACH STATEMENT
EXECUTE PROCEDURE refresh_materialized_view();
REFRESH MATERIALIZED VIEW tsvector
""")
# materialized_view_factory.py
from sqlalchemy.ext import compiler
from sqlalchemy.schema import DDLElement
from bootstrap import db
class CreateMaterializedView(DDLElement):
def __init__(self, name, selectable):
self.name = name
self.selectable = selectable
@compiler.compiles(CreateMaterializedView)
def compile(element, compiler, **kw):
# Could use "CREATE OR REPLACE MATERIALIZED VIEW..."
# but I'd rather have noisy errors
return 'CREATE MATERIALIZED VIEW {} AS {}'.format(
element.name,
compiler.sql_compiler.process(element.selectable, literal_binds=True))
def create_mat_view(name, selectable, metadata=db.metadata):
"""Creates a materialized view table"""
_mt = db.MetaData() # temp metadata just for initial Table object creation
t = db.Table(name, _mt) # the actual mat view class is bound to db.metadata
for c in selectable.c:
t.append_column(db.Column(c.name, c.type, primary_key=c.primary_key))
db.event.listen(
metadata, 'after_create',
CreateMaterializedView(name, selectable)
)
db.event.listen(
metadata, 'before_drop',
db.DDL('DROP MATERIALIZED VIEW IF EXISTS ' + name)
)
return t
......@@ -28,6 +28,7 @@ from .attribute import Attribute
from .attribute_picture import AttributePicture
from .shelter_picture import ShelterPicture
from .shelter_document import ShelterDocument
from .tsvector import Tsvector
from .page import Page
from .translation import Translation
......@@ -35,7 +36,7 @@ from .translation import Translation
__all__ = ['User', 'Shelter', 'Section', 'Property', 'Category', 'Attribute',
'Value', 'Page', 'Translation', 'AttributePicture',
'ShelterPicture', 'ShelterDocument', 'Association']
'ShelterPicture', 'ShelterDocument', 'Association', 'Tsvector']
import os
......
from bootstrap import db
from web.materialized_view_factory import create_mat_view
from web.models import Shelter, Property, Value, Attribute, Association
from sqlalchemy import func
#query = create_mat_view('value_lex', db.select([Property.shelter_id.label('shelter_id'),
# func.to_tsvector(func.string_agg(Value.name,' ')).label("lexemes")]
# ).select_from(db.join(Property, Attribute, Property.attribute_id==Attribute.id)#, isouter=True)
# ).group_by(Property.shelter_id))
#print(query.c.shelter_id)
class Tsvector(db.Model):
__table__ = create_mat_view("tsvector",
db.select([Shelter.id.label('shelter_id'),
func.to_tsvector(func.string_agg(Value.name,' ')).label("lexeme")])\
.select_from(db.join(db.join(db.join(Shelter,Property), Association), Value))\
.group_by(Shelter.id, Property.shelter_id))
# document = db.session.query(Property.shelter_id,func.to_tsvector(func.string_agg(Value.name,' ')).label("text"))\
# .join(Attribute)\
# .join(Association,Property.id==Association.property_id)\
# .join(Value, Association.value_id==Value.id)\
# .group_by(Property.shelter_id)\
# .subquery()
......@@ -19,7 +19,7 @@ from bootstrap import db
from sqlalchemy.sql import func, select
from flask import Blueprint, jsonify, request
from collections import defaultdict
from web.models import Shelter, Attribute, Property, Value, Association, ShelterPicture, Category
from web.models import Shelter, Attribute, Property, Value, Association, ShelterPicture, Category, Tsvector
api_bp = Blueprint('api for shelter', __name__, url_prefix='/api/v0.1')
......@@ -180,19 +180,19 @@ def attributes(attribute_name, attribute_value=''):
@api_bp.route('/shelters/search/<searchstring>', methods=['GET'])
def fulltext(searchstring):
searchstring = request.args.get('string')
print(searchstring)
"""Returns shelter id's which match the search criteria. Uses full text search'"""
result= tree()
document = db.session.query(Property.shelter_id,func.to_tsvector(func.string_agg(Value.name,' ')).label("text"))\
.join(Attribute)\
.join(Association,Property.id==Association.property_id)\
.join(Value, Association.value_id==Value.id)\
.group_by(Property.shelter_id)\
.subquery()
searchquery = db.session.query(document).filter(document.c.text.match(searchstring))
# document = db.session.query(Property.shelter_id,func.to_tsvector(func.string_agg(Value.name,' ')).label("text"))\
# .join(Attribute)\
# .join(Association,Property.id==Association.property_id)\
# .join(Value, Association.value_id==Value.id)\
# .group_by(Property.shelter_id)\
# .subquery()
# searchquery = db.session.query(document).filter(document.c.text.match(searchstring))
searchquery = Tsvector.query.filter(Tsvector.lexeme.match(searchstring))
#print(searchquery)
for shelter_property in searchquery:
#print(shelter_property)
result[shelter_property.shelter_id] = 1#shelter_property.text
result[shelter_property.shelter_id]
return jsonify(result)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment