Initial commit.

parents
# use glob syntax
syntax: glob
*.elc
*.pyc
*~
*.db
# Emacs
eproject.cfg
# Temporary files (vim backups)
*.swp
# Log files:
*.log
# Vagrant:
.vagrant/
# Virtualenv
venv
build
conf/conf.cfg
.coverage
# node files
node_modules
Cédric Bonhomme <cedric@cedricbonhomme.org>, Luxembourg Institute of Science and Technology
Shelter Database project news
0.1 (2015-03-30)
- initial release.
Shelter Database
================
# Presentation
Shelter Database.
# Deployment
Tested with Python 3.5 and Python 2.7.
## Configure the database
$ sudo apt-get install -y postgresql postgresql-server-dev-9.4 postgresql-client
$ echo "127.0.0.1:5433:shelter:pgsqluser:pgsqlpwd" > ~/.pgpass
$ chmod 0600 ~/.pgpass
$ sudo -u postgres createuser shelter --no-superuser --createdb --no-createrole
$ sudo -u postgres createdb shelter --no-password
$ echo "ALTER USER pgsqluser WITH ENCRYPTED PASSWORD 'pgsqlpwd';" | sudo -u postgres psql
$ echo "GRANT ALL PRIVILEGES ON DATABASE shelter TO pgsqluser;" | sudo -u postgres psql
## Get the code and configure the database connection
$ git clone https://git.list.lu/
$ cd shelter-database/
$ cp conf/conf.cfg-sample conf/conf.cfg
If you have chosen another database name, username or password:
$ sed -i '/database/d' conf/conf.cfg
$ sed -i '/database_url/d' conf/conf.cfg
$ echo '[database]' >> conf/conf.cfg
$ echo 'database_url = postgres://pgsqluser:pgsqlpwd@127.0.0.1:5433/warroom' >> conf/conf.cfg
Update appropriately the line just above.
## Install the Python requirements
$ sudo apt-get install python-pip python-dev
$ sudo pip install --upgrade -r requirements.txt
## Install the JavaScript requirements
$ sudo apt-get install nodejs-legacy
$ sudo npm install -g bower
$ bower install
## Launch the application and populate the database
$ nohup python runserver.py &
$ ./init_db.sh
# Generation of the database UML graph
$ python manager.py uml_graph
# Example of requests to the Web Service
$ GET http://127.0.0.1:5000/api/category?q={"filters":[{"name":"parent_id","op":"is_null"}]}
$ GET http://127.0.0.1:5000/api/category?q={"filters":[{"name":"parent_id","op":"is_not_null"}]}
$ GET http://127.0.0.1:5000/api/category?q={"filters":[{"name":"parent_id","op":"is_not_null"},{"name":"name","op":"eq","val":"Walls %26 frame"}]}
$ GET http://127.0.0.1:5000/api/attribute?q={"filters":[{"name":"name","op":"eq","val":"Landform"}]}
$ GET http://127.0.0.1:5000/api/attribute?q={"filters":[{"name":"name","op":"eq","val":"Main hazards in country"}]}
# License
# Contact
#! /usr/bin/env python
#-*- coding: utf-8 -*-
# ***** BEGIN LICENSE BLOCK *****
# This file is part of Shelter Database.
# Copyright (c) 2016
# All rights reserved.
#
#
#
# ***** END LICENSE BLOCK *****
__author__ = "Cedric Bonhomme"
__version__ = "$Revision: 0.1 $"
__date__ = "$Date: 2016/03/30$"
__revision__ = "$Date: 2016/03/30 $"
__copyright__ = "Copyright (c) "
__license__ = ""
"""Bootstrap
Required imports and code execution for basic functionning.
"""
import os
import sys
import logging
import conf
from flask import Flask
from flask.ext.sqlalchemy import SQLAlchemy
import flask.ext.restless
# Create Flask app
app = Flask('web')
# Create a random secrey key so we can use sessions
app.config['SECRET_KEY'] = os.urandom(12)
app.debug = conf.LOG_LEVEL <= logging.DEBUG
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = True
app.config['SQLALCHEMY_DATABASE_URI'] = conf.SQLALCHEMY_DATABASE_URI
db = SQLAlchemy(app)
# Create the Flask-Restless API manager.
manager = flask.ext.restless.APIManager(app, flask_sqlalchemy_db=db)
def populate_g():
from flask import g
g.db = db
g.app = app
#! /usr/bin/env python
#-*- coding: utf-8 -*-
# ***** BEGIN LICENSE BLOCK *****
# This file is part of LPS - Abstraction Layer.
# Copyright (c) 2015-2016 Luxembourg Institute of Science and Technology.
# All rights reserved.
#
#
#
# ***** END LICENSE BLOCK *****
__author__ = "Cedric Bonhomme"
__version__ = "$Revision: 0.1 $"
__date__ = "$Date: 2015/12/09$"
__revision__ = "$Date: 2015/12/09 $"
__copyright__ = "Copyright (c) Luxembourg Institute of Science and Technology"
__license__ = ""
""" Program variables.
This file contain the variables used by the application.
"""
import os
import sys
import logging
try:
import configparser as confparser
except:
import ConfigParser as confparser
BASE_DIR = os.path.abspath(os.path.dirname(__file__))
PATH = os.path.abspath(".")
DEFAULTS = {"platform_url": "http://127.0.0.1:5000",
"host": "0.0.0.0",
"port": "5000",
"https": "false",
"debug": "true",
"log_path": "warroom.log",
"log_level": "info"
}
config = confparser.SafeConfigParser(defaults=DEFAULTS)
config.read(os.path.join(BASE_DIR, "conf/conf.cfg"))
PLATFORM_URL = config.get('misc', 'platform_url')
SQLALCHEMY_DATABASE_URI = config.get('database', 'database_url')
WEBSERVER_DEBUG = config.getboolean('webserver', 'debug')
WEBSERVER_HOST = config.get('webserver', 'host')
WEBSERVER_PORT = config.getint('webserver', 'port')
WEBSERVER_HTTPS = config.getboolean('webserver', 'https')
LOG_PATH = config.get('misc', 'log_path')
LOG_LEVEL = {'debug': logging.DEBUG,
'info': logging.INFO,
'warn': logging.WARN,
'error': logging.ERROR,
'fatal': logging.FATAL}[config.get('misc', 'log_level')]
[misc]
platform_url = http://127.0.0.1:5000/
log_path = ./web/var/shelter.log
log_level = info
[webserver]
debug = false
host = 0.0.0.0
port = 5000
https = false
[database]
database_url = postgres://pgsqluser:pgsqlpwd@127.0.0.1:5432/shelter
This diff is collapsed.
File added
This diff is collapsed.
#! /bin/sh
python3.5 manager.py db_empty
python3.5 manager.py db_create
python3.5 manager.py init_db
python3.5 manager.py populate_shelters
#! /usr/bin/env python
#-*- coding: utf-8 -*-
# ***** BEGIN LICENSE BLOCK *****
# This file is part of Shelter Database.
# Copyright (c) 2016
# All rights reserved.
#
#
#
# ***** END LICENSE BLOCK *****
__author__ = "Cedric Bonhomme"
__version__ = "$Revision: 0.1 $"
__date__ = "$Date: 2016/03/30$"
__revision__ = "$Date: 2016/03/30 $"
__copyright__ = "Copyright (c) "
__license__ = ""
from bootstrap import app, db, populate_g, conf
from flask.ext.script import Manager
import scripts
import web.models
manager = Manager(app)
@manager.command
def uml_graph():
"UML graph from the models."
with app.app_context():
web.models.uml_graph(db)
@manager.command
def db_empty():
"Will drop the database."
with app.app_context():
populate_g()
web.models.db_empty(db)
@manager.command
def db_create():
"Will create the database."
with app.app_context():
populate_g()
web.models.db_create(db)
@manager.command
def init_db():
"Will initialize the database with the attribute for the shelters."
with app.app_context():
scripts.init_db()
@manager.command
def init_db():
"Will import the shelters in the database."
with app.app_context():
scripts.populate_shelters()
if __name__ == '__main__':
manager.run()
#! /usr/bin/env python
#-*- coding: utf-8 -*-
# ***** BEGIN LICENSE BLOCK *****
# This file is part of Shelter Database.
# Copyright (c) 2016
# All rights reserved.
#
#
#
# ***** END LICENSE BLOCK *****
__author__ = "Cedric Bonhomme"
__version__ = "$Revision: 0.1 $"
__date__ = "$Date: 2016/03/30$"
__revision__ = "$Date: 2016/03/30 $"
__copyright__ = "Copyright (c) "
__license__ = ""
from bootstrap import conf, app, manager
from web import models
#from web import websocketprocessors as processors
with app.app_context():
# Views to render the HTML files
from web import views
# 'User' Web service
blueprint_user = manager.create_api_blueprint(models.User,
methods=['GET', 'POST', 'PUT', 'DELETE'])
app.register_blueprint(blueprint_user)
# 'Shelter' Web service
blueprint_shelter = manager.create_api_blueprint(models.Shelter,
methods=['GET', 'POST', 'PUT', 'DELETE'])
app.register_blueprint(blueprint_shelter)
# 'Category' Web service
blueprint_category = manager.create_api_blueprint(models.Category,
methods=['GET', 'POST', 'PUT', 'DELETE'])
app.register_blueprint(blueprint_category)
# 'Attribute' Web service
blueprint_attribute = manager.create_api_blueprint(models.Attribute,
methods=['GET', 'POST', 'PUT', 'DELETE'])
app.register_blueprint(blueprint_attribute)
if __name__ == "__main__":
app.run(host=conf.WEBSERVER_HOST,
port=conf.WEBSERVER_PORT,
debug=conf.WEBSERVER_DEBUG)
#! /usr/bin/env python
# -*- coding: utf-8 -*-
from .init_db import init_db
__all__ = ['init_db']
#! /usr/bin/python
#-*- coding:utf-8 -*
import csv
from web import models
from bootstrap import db
def init_db():
print("Importing base structure of shelters...")
with open('data/Shelters_Structure.csv', newline='') as csvfile:
structure = csv.reader(csvfile, delimiter=',')
category = None
sub_category = None
sub_category_name = ''
for index, row in enumerate(structure):
if index == 0:
continue
if row[0] != '':
category_name = row[0]
category = models.Category(name=category_name)
db.session.add(category)
db.session.commit()
if sub_category_name != row[1]:
sub_category_name = row[1]
sub_category = models.Category(name=sub_category_name)
db.session.add(sub_category)
category.sub_categories.append(sub_category)
db.session.commit()
attribute_name = row[2]
attribute = models.Attribute(name=attribute_name,
category_id=sub_category.id)
attribute_type = row[3] # TODO: do something with row[3] ...
cardinality = row[5]
attribute.multiple = cardinality=='multiple choice'
#if cardinality == '':
# free text for the value of this attribute
if cardinality in ('single choice', 'multiple choice'):
for value in row[4].split(';'):
value_name = value.strip()
value = models.Value(name=value_name,
attribute_id=attribute.id)
db.session.add(value)
attribute.associated_values.append(value)
db.session.add(attribute)
db.session.commit()
#images = row[7]
#! /usr/bin/python
#-*- coding:utf-8 -*
import csv
from web import models
from bootstrap import db
def init_db():
print("Importing base structure of shelters...")
with open('data/Shelters_Structure.csv', newline='') as csvfile:
from .login import LoginForm
#! /usr/bin/env python
#-*- coding: utf-8 -*-
# ***** BEGIN LICENSE BLOCK *****
# This file is part of Shelter Database.
# Copyright (c) 2016
# All rights reserved.
#
#
#
# ***** END LICENSE BLOCK *****
__author__ = "Cedric Bonhomme"
__version__ = "$Revision: 0.1 $"
__date__ = "$Date: 2016/03/30$"
__revision__ = "$Date: 2016/03/30 $"
__copyright__ = "Copyright (c) "
__license__ = ""
from flask.ext.wtf import Form
from flask import url_for, redirect
from wtforms import validators, TextField, PasswordField, BooleanField, \
SubmitField, HiddenField
from web.models import User
from web.lib import utils
class RedirectForm(Form):
"""
Secure back redirects with WTForms.
"""
next = HiddenField()
def __init__(self, *args, **kwargs):
Form.__init__(self, *args, **kwargs)
if not self.next.data:
self.next.data = utils.get_redirect_target() or ''
def redirect(self, endpoint='start', **values):
if utils.is_safe_url(self.next.data):
return redirect(self.next.data)
target = utils.get_redirect_target()
return redirect(target or url_for(endpoint, **values))
class LoginForm(RedirectForm):
"""
Login form.
"""
name = TextField("Name",
[validators.Required("Please enter your name.")])
password = PasswordField('Password',
[validators.Required("Please enter a password.")])
remember_me = BooleanField("Remember me", default=False)
submit = SubmitField("Log In")
def validate(self):
if not super(LoginForm, self).validate():
return False
user = User.query.filter(User.name==self.name.data).first()
if user and user.check_password(self.password.data):
return True
else:
self.name.errors.append("Invalid name or password")
return False
#! /usr/bin/env python
# -*- coding: utf-8 -*-
# ***** BEGIN LICENSE BLOCK *****
# This file is part of LPS - Abstraction Layer.
# Copyright (c) 2015-2016 Luxembourg Institute of Science and Technology.
# All rights reserved.
#
#
#
# ***** END LICENSE BLOCK *****
__author__ = "Cedric Bonhomme"
__version__ = "$Revision: 0.1 $"
__date__ = "$Date: 2016/03/01$"
__revision__ = "$Date: 2016/03/01 $"
__copyright__ = "Copyright (c) Luxembourg Institute of Science and Technology"
__license__ = ""
from flask import request
try:
from urlparse import urlparse, parse_qs, urljoin
except:
from urllib.parse import urlparse, parse_qs, urljoin
def is_safe_url(target):
"""
Ensures that a redirect target will lead to the same server.
"""
ref_url = urlparse(request.host_url)
test_url = urlparse(urljoin(request.host_url, target))
return test_url.scheme in ('http', 'https') and \
ref_url.netloc == test_url.netloc
def get_redirect_target():
"""
Looks at various hints to find the redirect target.
"""
for target in request.args.get('next'), request.referrer:
if not target:
continue
if is_safe_url(target):
return target
#! /usr/bin/env python
#-*- coding: utf-8 -*-
# ***** BEGIN LICENSE BLOCK *****
# This file is part of Shelter Database.
# Copyright (c) 2016
# All rights reserved.
#
#
#
# ***** END LICENSE BLOCK *****
__author__ = "Cedric Bonhomme"
__version__ = "$Revision: 0.1 $"
__date__ = "$Date: 2016/03/30$"
__revision__ = "$Date: 2016/03/30 $"
__copyright__ = "Copyright (c) "
__license__ = ""
from .user import User
from .shelter import Shelter
from .property import Property
from .category import Category
from .attribute import Attribute
from .value import Value
__all__ = ['User', 'Shelter', 'Property', 'Category', 'Attribute', 'Value']
import os
from sqlalchemy.engine import reflection
from sqlalchemy.schema import (
MetaData,
Table,
DropTable,
ForeignKeyConstraint,
DropConstraint)
def mappers(*args):
from sqlalchemy.orm import class_mapper
return [class_mapper(x) for x in args]
def uml_graph(db):
"""Generate a UML diagram from the models."""
import sqlalchemy_schemadisplay as sasd
graph = sasd.create_uml_graph(
mappers(User, Shelter, Property,
Category, Attribute, Value),
show_operations=False,
show_multiplicity_one=True
)
graph.write_png('uml_graph.png') # write out the file
def db_empty(db):
"Will drop every datas stocked in db."
# From http://www.sqlalchemy.org/trac/wiki/UsageRecipes/DropEverything
conn = db.engine.connect()
# the transaction only applies if the DB supports
# transactional DDL, i.e. Postgresql, MS SQL Server
trans = conn.begin()
inspector = reflection.Inspector.from_engine(db.engine)
# gather all data first before dropping anything.
# some DBs lock after things have been dropped in
# a transaction.
metadata = MetaData()
tbs = []
all_fks = []
for table_name in inspector.get_table_names():
fks = []