Commit 422b146b authored by Damien Octeau's avatar Damien Octeau Committed by GitHub

Merge pull request #1 from cedricbonhomme/master

Refactoring in order to isolate PRIMO to its own namespace.
parents b4b3ae56 d154ee86
# Created by https://www.gitignore.io/api/python
### Python ###
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
*.c
# Distribution / packaging
.Python
env/
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
*.egg-info/
.installed.cfg
*.egg
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*,cover
.hypothesis/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# IPython Notebook
.ipynb_checkpoints
# pyenv
.python-version
# celery beat schedule file
celerybeat-schedule
# dotenv
.env
# virtualenv
venv/
ENV/
# Spyder project settings
.spyderproject
# Rope project settings
.ropeproject
......@@ -15,13 +15,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#exclude linking/*.pyx
#exclude primo/linking/*.pyx
include *.txt
include primo.py
include make_plots_and_stats.py
include performance_experiments.py
include bin/primo
include bin/make_plots_and_stats
include bin/performance_experiments
include setup.py
include linking/*.c
include linking/*.pxd
include linking/*.pxi
include linking/*.py
include primo/linking/*.c
include primo/linking/*.pxd
include primo/linking/*.pxi
include primo/linking/*.py
The PRIMO tool for static Intent resolution and probabilistic value inference.
# Installation
```shell
$ sudo python setup.py build
$ sudo python setup.py install
```
The script can be run directly from the top-level directory afterwards.
For further instructions, please see http://siis.cse.psu.edu/primo
# Usage
## As an application
```shell
$ primo --protodir ic3-output/ --dumpintentlinks links.blp
$ mkdir stats
$ make_plots_and_stats --input links.blp --out stats/
2016-05-10 09:20:34,538 [root] [INFO ] Loading file links.blp.
2016-05-10 09:20:34,541 [root] [INFO ] Loaded 35375 Intent links.
2016-05-10 09:20:34,541 [root] [INFO ] Plotting probability distribution CDF.
2016-05-10 09:20:35,669 [root] [INFO ] Finished plotting probability distribution CDF.
2016-05-10 09:20:35,669 [root] [INFO ] Plotting probability distribution histogram.
2016-05-10 09:20:36,632 [root] [INFO ] Finished plotting probability distribution histogram.
2016-05-10 09:20:36,632 [root] [INFO ] Plotting probability distribution by inter/intra app Intent.
2016-05-10 09:20:41,133 [root] [INFO ] Finished plotting histograms.
2016-05-10 09:20:41,133 [root] [INFO ] Computing Intent connectivities.
2016-05-10 09:20:41,158 [root] [INFO ] Finished computing Intent connectivities from Intent links.
2016-05-10 09:20:41,159 [root] [INFO ] Computing connectivity CDFs.
2016-05-10 09:20:41,731 [root] [INFO ] Plotting link CDF.
2016-05-10 09:20:42,272 [root] [INFO ] Finished plotting link CDF.
```
## As a module
```python
#! /usr/bin/env python
# -*- coding: utf-8 -*-
import sys
from primo.linking.fetch_data import FetchData
if __name__ == "__main__":
# Point of entry in execution mode
file_path = sys.argv[1:]
apps, components, intents, intent_filters = FetchData(file_path, "", True)
print("")
for app in apps:
print(app.name)
print("""\tSends external intent: {}
Has publicly accessible component: {}""". \
format(app.SendsExternalIntent(),
app.HasOpenComponent()))
```
To build:
python setup.py build
The script can be run directly from the top-level directory afterwards.
It can also be installed with python setup.py install
For further instructions, please see http://siis.cse.psu.edu/primo/
......@@ -25,7 +25,7 @@ import sys
import gflags
from linking.plots_and_stats import MakePlotsAndStats
from primo.linking.plots_and_stats import MakePlotsAndStats
FLAGS = gflags.FLAGS
......
......@@ -29,7 +29,7 @@ import time
import gflags
from linking.find_links import FindLinksAndLogExceptions
from primo.linking.find_links import FindLinksAndLogExceptions
LOGGER = logging.getLogger(__name__)
......
......@@ -36,7 +36,7 @@ import time
import gflags
from linking import find_links
from primo.linking import find_links
FLAGS = gflags.FLAGS
......
......@@ -15,12 +15,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Application class and factory."""
from linking.components cimport Component
from linking.components cimport MakeComponent
from linking.intents cimport ComponentIntent
from linking.attribute_matching cimport AttributeMap
from primo.linking.components cimport Component
from primo.linking.components cimport MakeComponent
from primo.linking.intents cimport ComponentIntent
from primo.linking.attribute_matching cimport AttributeMap
from linking import ic3_data_pb2
from primo.linking import ic3_data_pb2
cdef set SAMPLES = set()
......
......@@ -15,7 +15,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from linking.applications cimport Application
from primo.linking.applications cimport Application
cdef int GetSkippedFilterCount()
cdef Component MakeComponent(object component_pb, Application application,
......
......@@ -16,11 +16,11 @@
# limitations under the License.
"""A class and factory for Android application components."""
from linking.applications cimport Application
from linking.target_data cimport AddComponent
from linking.intent_filters cimport IntentFilter
from linking.intent_filters cimport MakeIntentFilter
from linking.intents cimport MakeComponentIntent
from primo.linking.applications cimport Application
from primo.linking.target_data cimport AddComponent
from primo.linking.intent_filters cimport IntentFilter
from primo.linking.intent_filters cimport MakeIntentFilter
from primo.linking.intents cimport MakeComponentIntent
cdef int _id = 0
......
......@@ -20,8 +20,8 @@ from google.protobuf import text_format
import logging
import os
from linking import applications
from linking import ic3_data_pb2
from primo.linking import applications
from primo.linking import ic3_data_pb2
LOGGER = logging.getLogger(__name__)
......
......@@ -16,8 +16,8 @@
# limitations under the License.
cimport numpy as np
from linking.components cimport Component
from linking.intents cimport Intent
from primo.linking.components cimport Component
from primo.linking.intents cimport Intent
ctypedef np.int8_t DTYPE_t
......
......@@ -24,15 +24,15 @@ import logging
import numpy as np
import time
from linking.applications cimport Application
from linking.components cimport Component
from linking.target_data cimport GetAppsMatching
from linking.target_data cimport GetComponentsOfApp
from linking.target_data cimport GetComponentsWithKind
from linking.target_data cimport GetComponentsWithName
from linking.target_data cimport GetExportedComponentCount
from linking.target_data cimport GetExportedComponents
from linking.intents cimport Intent
from primo.linking.applications cimport Application
from primo.linking.components cimport Component
from primo.linking.target_data cimport GetAppsMatching
from primo.linking.target_data cimport GetComponentsOfApp
from primo.linking.target_data cimport GetComponentsWithKind
from primo.linking.target_data cimport GetComponentsWithName
from primo.linking.target_data cimport GetExportedComponentCount
from primo.linking.target_data cimport GetExportedComponents
from primo.linking.intents cimport Intent
DTYPE = np.int8
......
......@@ -16,9 +16,9 @@
# limitations under the License.
cimport numpy as np
from linking.intent_filters cimport IntentFilter
from linking.intents cimport ComponentIntent
from linking.intents cimport Intent
from primo.linking.intent_filters cimport IntentFilter
from primo.linking.intents cimport ComponentIntent
from primo.linking.intents cimport Intent
ctypedef np.int8_t DTYPE_t
......
......@@ -21,36 +21,36 @@ DEF DEBUG = False
cimport cython
cimport numpy as np
from linking.target_data import BASE_TYPE
from linking.util import Powerset
from primo.linking.target_data import BASE_TYPE
from primo.linking.util import Powerset
import logging
import numpy as np
import time
from linking.intent_data cimport GetAttributeMaps
from linking.intent_data cimport GetPreciseIntents
from linking.target_data cimport GetExportedFilters
from linking.target_data cimport GetFiltersOfApp
from linking.target_data cimport GetFiltersWithUsedPermission
from linking.target_data cimport GetFiltersWithAction
from linking.target_data cimport GetFiltersWithAnyAction
from linking.target_data cimport GetFiltersWithBaseTypes
from linking.target_data cimport GetFiltersWithCategories
from linking.target_data cimport GetFiltersWithHost
from linking.target_data cimport GetFiltersWithKind
from linking.target_data cimport GetFiltersWithPath
from linking.target_data cimport GetFiltersWithPort
from linking.target_data cimport GetFiltersWithScheme
from linking.target_data cimport GetFiltersWithType
from linking.target_data cimport GetFiltersWithTypes
from linking.target_data cimport GetFiltersWithoutType
from linking.target_data cimport GetNoDataFilters
from linking.intent_filters cimport IntentFilter
from linking.intents cimport ComponentIntent
from linking.intents cimport Intent
from linking.attribute_matching cimport NonEmptyIntersection
include 'linking/constants.pxi'
from primo.linking.intent_data cimport GetAttributeMaps
from primo.linking.intent_data cimport GetPreciseIntents
from primo.linking.target_data cimport GetExportedFilters
from primo.linking.target_data cimport GetFiltersOfApp
from primo.linking.target_data cimport GetFiltersWithUsedPermission
from primo.linking.target_data cimport GetFiltersWithAction
from primo.linking.target_data cimport GetFiltersWithAnyAction
from primo.linking.target_data cimport GetFiltersWithBaseTypes
from primo.linking.target_data cimport GetFiltersWithCategories
from primo.linking.target_data cimport GetFiltersWithHost
from primo.linking.target_data cimport GetFiltersWithKind
from primo.linking.target_data cimport GetFiltersWithPath
from primo.linking.target_data cimport GetFiltersWithPort
from primo.linking.target_data cimport GetFiltersWithScheme
from primo.linking.target_data cimport GetFiltersWithType
from primo.linking.target_data cimport GetFiltersWithTypes
from primo.linking.target_data cimport GetFiltersWithoutType
from primo.linking.target_data cimport GetNoDataFilters
from primo.linking.intent_filters cimport IntentFilter
from primo.linking.intents cimport ComponentIntent
from primo.linking.intents cimport Intent
from primo.linking.attribute_matching cimport NonEmptyIntersection
include 'primo/linking/constants.pxi'
DTYPE = np.int8
......
......@@ -24,18 +24,18 @@ import traceback
import gflags
from linking.find_explicit_links cimport ExplicitLinkFinder
from linking.find_implicit_links cimport ImplicitLinkFinder
from linking.target_data cimport PrepareForQueries
from linking.intent_data cimport GetImpreciseComponentIntents
from linking.intent_data cimport GetPreciseComponentIntents
from linking.intents cimport ComponentIntent
from linking.intents cimport Intent
from linking.validation cimport PerformValidation
from linking import fetch_data
from linking import intents as intents_mod
from linking import write_results
from primo.linking.find_explicit_links cimport ExplicitLinkFinder
from primo.linking.find_implicit_links cimport ImplicitLinkFinder
from primo.linking.target_data cimport PrepareForQueries
from primo.linking.intent_data cimport GetImpreciseComponentIntents
from primo.linking.intent_data cimport GetPreciseComponentIntents
from primo.linking.intents cimport ComponentIntent
from primo.linking.intents cimport Intent
from primo.linking.validation cimport PerformValidation
from primo.linking import fetch_data
from primo.linking import intents as intents_mod
from primo.linking import write_results
gflags.DEFINE_boolean('computeattributes', True, 'Compute attributes.')
......
......@@ -15,7 +15,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from linking.intents cimport ComponentIntent
from primo.linking.intents cimport ComponentIntent
cdef void AddPreciseIntent(ComponentIntent intent)
cdef void AddImpreciseIntent(ComponentIntent intent)
......
......@@ -16,17 +16,17 @@
# limitations under the License.
"""Global Intent maps and constants."""
from linking.intents cimport ComponentIntent
from linking.intents cimport Intent
from primo.linking.intents cimport ComponentIntent
from primo.linking.intents cimport Intent
import logging
from linking.target_data import BASE_TYPE
from linking.target_data import CLASS
from linking.target_data import PACKAGE
from linking import ic3_data_pb2
from primo.linking.target_data import BASE_TYPE
from primo.linking.target_data import CLASS
from primo.linking.target_data import PACKAGE
from primo.linking import ic3_data_pb2
include 'linking/constants.pxi'
include 'primo/linking/constants.pxi'
LOGGER = logging.getLogger(__name__)
......
......@@ -15,7 +15,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from linking.components cimport Component
from primo.linking.components cimport Component
cdef IntentFilter MakeIntentFilter(object intent_filter_pb,
Component component)
......
......@@ -16,10 +16,10 @@
# limitations under the License.
"""Class and factory for Intent Filters."""
from linking.components cimport Component
from linking.target_data cimport AddIntentFilterAttributes
from primo.linking.components cimport Component
from primo.linking.target_data cimport AddIntentFilterAttributes
from linking import ic3_data_pb2
from primo.linking import ic3_data_pb2
_ACTION_MAIN = u'android.intent.action.MAIN'
......@@ -170,7 +170,7 @@ cdef class IntentFilter(object):
"""
return (self.actions and len(self.actions) >= 1
and IntentFilter._ACTION_MAIN in self.actions)
_ACTION_MAIN in self.actions)
def print_end_point(self):
print "package",self.component.application_id
......
......@@ -15,7 +15,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from linking.intents cimport Intent
from primo.linking.intents cimport Intent
cdef void UpdateImpreciseDistribution(Intent intent)
cdef void MakeRandomImprecision(Intent intent)
......@@ -17,12 +17,12 @@
"""Module for recording field imprecision distribution and for introducing
imprecise fields according to a given distribution."""
from linking.intents cimport Intent
from primo.linking.intents cimport Intent
from collections import Counter
import random
include 'linking/constants.pxi'
include 'primo/linking/constants.pxi'
# Explicit Intent imprecision counters.
......
......@@ -15,8 +15,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from linking.applications cimport Application
from linking.components cimport Component
from primo.linking.applications cimport Application
from primo.linking.components cimport Component
cdef class Intent(object):
cdef readonly unicode permission
......
......@@ -16,21 +16,21 @@
# limitations under the License.
"""Class and factory for Intent."""
from linking.applications cimport Application
from linking.components cimport Component
from linking.intent_data cimport AddImpreciseIntent
from linking.intent_data cimport AddPreciseIntent
from linking.target_data cimport GetTargetCountForValue
from primo.linking.applications cimport Application
from primo.linking.components cimport Component
from primo.linking.intent_data cimport AddImpreciseIntent
from primo.linking.intent_data cimport AddPreciseIntent
from primo.linking.target_data cimport GetTargetCountForValue
from collections import Counter
from linking import ic3_data_pb2
from linking.target_data import CLASS
from linking.target_data import KIND
from linking.target_data import PACKAGE
from linking.target_data import USED_PERMISSIONS
from primo.linking import ic3_data_pb2
from primo.linking.target_data import CLASS
from primo.linking.target_data import KIND
from primo.linking.target_data import PACKAGE
from primo.linking.target_data import USED_PERMISSIONS
include 'linking/constants.pxi'
include 'primo/linking/constants.pxi'
cdef dict _INTENT_COUNTERS = {ic3_data_pb2.ACTION: Counter(),
......
......@@ -34,7 +34,7 @@ from matplotlib import rcParams
import bloscpack as bp
import numpy as np
from linking.write_results cimport Row
from primo.linking.write_results cimport Row
FONT_SIZE = 22
......
......@@ -15,9 +15,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from linking.applications cimport Application
from linking.components cimport Component
from linking.intent_filters cimport IntentFilter
from primo.linking.applications cimport Application
from primo.linking.components cimport Component
from primo.linking.intent_filters cimport IntentFilter
cdef void PrepareForQueries(set applications)
cdef int GetExportedComponentCount(int kind, set search_space=?)
......
......@@ -23,10 +23,10 @@ DEF DYNAMIC_RECEIVER = 3
from collections import Counter
import logging
from linking.attribute_matching cimport AttributeMap
from primo.linking.attribute_matching cimport AttributeMap
from linking.util import Powerset
from linking import ic3_data_pb2
from primo.linking.util import Powerset
from primo.linking import ic3_data_pb2
LOGGER = logging.getLogger(__name__)
......
......@@ -20,20 +20,20 @@ Goodman-Kruskal's gamma."""
cimport cython
cimport numpy as np
from linking.components cimport GetSkippedFilterCount
from linking.intent_data cimport AddPreciseIntent
from linking.intent_data cimport AddImpreciseIntent
from linking.intent_imprecisions cimport MakeRandomImprecision
from linking.intent_imprecisions cimport UpdateImpreciseDistribution
from linking.intents cimport ComponentIntent
from primo.linking.components cimport GetSkippedFilterCount
from primo.linking.intent_data cimport AddPreciseIntent
from primo.linking.intent_data cimport AddImpreciseIntent
from primo.linking.intent_imprecisions cimport MakeRandomImprecision
from primo.linking.intent_imprecisions cimport UpdateImpreciseDistribution
from primo.linking.intents cimport ComponentIntent
import logging
import numpy as np
import gflags
from linking import intent_data
import linking.find_links
from primo.linking import intent_data
import primo.linking.find_links
FLAGS = gflags.FLAGS
......@@ -72,7 +72,7 @@ cdef void PerformValidation(list intents, bint skip_empty, set components,
# The ground truth contains all links with "full confidence" (priority = 100).
# Any link not in this set has priority 0.
ground_truth, _, _, _, _, _, _ = linking.find_links.FindLinksForIntents(
ground_truth, _, _, _, _, _, _ = primo.linking.find_links.FindLinksForIntents(
precise, set(), skip_empty, components, intent_filters, False, False)
# Store the ground truth targets into a set for efficient lookup.
......@@ -135,7 +135,7 @@ cdef void PerformValidation(list intents, bint skip_empty, set components,
still_precise += 1
validation = new_validation
intent_links, _, _, _, _, _, _ = linking.find_links.FindLinksForIntents(
intent_links, _, _, _, _, _, _ = primo.linking.find_links.FindLinksForIntents(
training, validation, skip_empty, components, intent_filters, True,
True)
......
......@@ -22,8 +22,8 @@ cimport numpy as np
import bloscpack
import numpy as np
from linking.intents cimport ComponentIntent
from linking.intents cimport Intent
from primo.linking.intents cimport ComponentIntent
from primo.linking.intents cimport Intent
DTYPE = [('intent', 'int32'),
......
......@@ -28,10 +28,10 @@ import unittest
sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)),
'..'))
from linking import write_results
from linking.applications import Application
from linking.intents import ComponentIntent
from linking.intents import Intent
from primo.linking import write_results
from primo.linking.applications import Application
from primo.linking.intents import ComponentIntent
from primo.linking.intents import Intent
class MockTarget(object):
......
......@@ -22,10 +22,12 @@ from distutils.core import setup
from distutils.extension import Extension
SRC_DIR = 'linking'
SRC_DIR = 'primo/linking'
if glob.glob(os.path.join(SRC_DIR, '*.pyx')):
if glob.glob(os.path.join(SRC_DIR, '*.c')):
use_cython = False
elif glob.glob(os.path.join(SRC_DIR, '*.pyx')):
try:
from Cython.Distutils import build_ext
use_cython = True
......@@ -63,8 +65,9 @@ EMAIL = 'octeau@cse.psu.edu'
LICENSE = 'Apache 2.0'
PACKAGES = [SRC_DIR]
SCRIPTS = ['primo.py', 'make_plots_and_stats.py', 'performance_experiments.py']
PACKAGES = ['primo', 'primo.linking']
SCRIPTS = ['bin/primo', 'bin/make_plots_and_stats',
'bin/performance_experiments']
CMD_CLASS = {}
OPTIONS = {}
......@@ -90,7 +93,7 @@ if __name__ == "__main__":
extension = '.c'
# Get the list of extensions.
ext_names = ScanDir('linking', extension)
ext_names = ScanDir('primo', extension)