Refactorization.

parent b4b3ae56
# Created by https://www.gitignore.io/api/python
### Python ###
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
env/
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
*.egg-info/
.installed.cfg
*.egg
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*,cover
.hypothesis/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# IPython Notebook
.ipynb_checkpoints
# pyenv
.python-version
# celery beat schedule file
celerybeat-schedule
# dotenv
.env
# virtualenv
venv/
ENV/
# Spyder project settings
.spyderproject
# Rope project settings
.ropeproject
......@@ -15,13 +15,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#exclude linking/*.pyx
#exclude primo/linking/*.pyx
include *.txt
include primo.py
include make_plots_and_stats.py
include performance_experiments.py
include bin/primo
include bin/make_plots_and_stats
include bin/performance_experiments
include setup.py
include linking/*.c
include linking/*.pxd
include linking/*.pxi
include linking/*.py
include primo/linking/*.c
include primo/linking/*.pxd
include primo/linking/*.pxi
include primo/linking/*.py
To build:
# Installation
python setup.py build
python setup.py build
The script can be run directly from the top-level directory afterwards.
It can also be installed with python setup.py install
The script can be run directly from the top-level directory afterwards.
For further instructions, please see http://siis.cse.psu.edu/primo/
......@@ -25,7 +25,7 @@ import sys
import gflags
from linking.plots_and_stats import MakePlotsAndStats
from primo.linking.plots_and_stats import MakePlotsAndStats
FLAGS = gflags.FLAGS
......
......@@ -29,7 +29,7 @@ import time
import gflags
from linking.find_links import FindLinksAndLogExceptions
from primo.linking.find_links import FindLinksAndLogExceptions
LOGGER = logging.getLogger(__name__)
......
......@@ -36,7 +36,7 @@ import time
import gflags
from linking import find_links
from primo.linking import find_links
FLAGS = gflags.FLAGS
......
......@@ -15,12 +15,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Application class and factory."""
from linking.components cimport Component
from linking.components cimport MakeComponent
from linking.intents cimport ComponentIntent
from linking.attribute_matching cimport AttributeMap
from primo.linking.components cimport Component
from primo.linking.components cimport MakeComponent
from primo.linking.intents cimport ComponentIntent
from primo.linking.attribute_matching cimport AttributeMap
from linking import ic3_data_pb2
from primo.linking import ic3_data_pb2
cdef set SAMPLES = set()
......
......@@ -15,7 +15,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from linking.applications cimport Application
from primo.linking.applications cimport Application
cdef int GetSkippedFilterCount()
cdef Component MakeComponent(object component_pb, Application application,
......
......@@ -16,11 +16,11 @@
# limitations under the License.
"""A class and factory for Android application components."""
from linking.applications cimport Application
from linking.target_data cimport AddComponent
from linking.intent_filters cimport IntentFilter
from linking.intent_filters cimport MakeIntentFilter
from linking.intents cimport MakeComponentIntent
from primo.linking.applications cimport Application
from primo.linking.target_data cimport AddComponent
from primo.linking.intent_filters cimport IntentFilter
from primo.linking.intent_filters cimport MakeIntentFilter
from primo.linking.intents cimport MakeComponentIntent
cdef int _id = 0
......
......@@ -20,8 +20,8 @@ from google.protobuf import text_format
import logging
import os
from linking import applications
from linking import ic3_data_pb2
from primo.linking import applications
from primo.linking import ic3_data_pb2
LOGGER = logging.getLogger(__name__)
......
......@@ -16,8 +16,8 @@
# limitations under the License.
cimport numpy as np
from linking.components cimport Component
from linking.intents cimport Intent
from primo.linking.components cimport Component
from primo.linking.intents cimport Intent
ctypedef np.int8_t DTYPE_t
......
......@@ -24,15 +24,15 @@ import logging
import numpy as np
import time
from linking.applications cimport Application
from linking.components cimport Component
from linking.target_data cimport GetAppsMatching
from linking.target_data cimport GetComponentsOfApp
from linking.target_data cimport GetComponentsWithKind
from linking.target_data cimport GetComponentsWithName
from linking.target_data cimport GetExportedComponentCount
from linking.target_data cimport GetExportedComponents
from linking.intents cimport Intent
from primo.linking.applications cimport Application
from primo.linking.components cimport Component
from primo.linking.target_data cimport GetAppsMatching
from primo.linking.target_data cimport GetComponentsOfApp
from primo.linking.target_data cimport GetComponentsWithKind
from primo.linking.target_data cimport GetComponentsWithName
from primo.linking.target_data cimport GetExportedComponentCount
from primo.linking.target_data cimport GetExportedComponents
from primo.linking.intents cimport Intent
DTYPE = np.int8
......
......@@ -16,9 +16,9 @@
# limitations under the License.
cimport numpy as np
from linking.intent_filters cimport IntentFilter
from linking.intents cimport ComponentIntent
from linking.intents cimport Intent
from primo.linking.intent_filters cimport IntentFilter
from primo.linking.intents cimport ComponentIntent
from primo.linking.intents cimport Intent
ctypedef np.int8_t DTYPE_t
......
......@@ -21,36 +21,36 @@ DEF DEBUG = False
cimport cython
cimport numpy as np
from linking.target_data import BASE_TYPE
from linking.util import Powerset
from primo.linking.target_data import BASE_TYPE
from primo.linking.util import Powerset
import logging
import numpy as np
import time
from linking.intent_data cimport GetAttributeMaps
from linking.intent_data cimport GetPreciseIntents
from linking.target_data cimport GetExportedFilters
from linking.target_data cimport GetFiltersOfApp
from linking.target_data cimport GetFiltersWithUsedPermission
from linking.target_data cimport GetFiltersWithAction
from linking.target_data cimport GetFiltersWithAnyAction
from linking.target_data cimport GetFiltersWithBaseTypes
from linking.target_data cimport GetFiltersWithCategories
from linking.target_data cimport GetFiltersWithHost
from linking.target_data cimport GetFiltersWithKind
from linking.target_data cimport GetFiltersWithPath
from linking.target_data cimport GetFiltersWithPort
from linking.target_data cimport GetFiltersWithScheme
from linking.target_data cimport GetFiltersWithType
from linking.target_data cimport GetFiltersWithTypes
from linking.target_data cimport GetFiltersWithoutType
from linking.target_data cimport GetNoDataFilters
from linking.intent_filters cimport IntentFilter
from linking.intents cimport ComponentIntent
from linking.intents cimport Intent
from linking.attribute_matching cimport NonEmptyIntersection
include 'linking/constants.pxi'
from primo.linking.intent_data cimport GetAttributeMaps
from primo.linking.intent_data cimport GetPreciseIntents
from primo.linking.target_data cimport GetExportedFilters
from primo.linking.target_data cimport GetFiltersOfApp
from primo.linking.target_data cimport GetFiltersWithUsedPermission
from primo.linking.target_data cimport GetFiltersWithAction
from primo.linking.target_data cimport GetFiltersWithAnyAction
from primo.linking.target_data cimport GetFiltersWithBaseTypes
from primo.linking.target_data cimport GetFiltersWithCategories
from primo.linking.target_data cimport GetFiltersWithHost
from primo.linking.target_data cimport GetFiltersWithKind
from primo.linking.target_data cimport GetFiltersWithPath
from primo.linking.target_data cimport GetFiltersWithPort
from primo.linking.target_data cimport GetFiltersWithScheme
from primo.linking.target_data cimport GetFiltersWithType
from primo.linking.target_data cimport GetFiltersWithTypes
from primo.linking.target_data cimport GetFiltersWithoutType
from primo.linking.target_data cimport GetNoDataFilters
from primo.linking.intent_filters cimport IntentFilter
from primo.linking.intents cimport ComponentIntent
from primo.linking.intents cimport Intent
from primo.linking.attribute_matching cimport NonEmptyIntersection
include 'primo/linking/constants.pxi'
DTYPE = np.int8
......
......@@ -24,18 +24,18 @@ import traceback
import gflags
from linking.find_explicit_links cimport ExplicitLinkFinder
from linking.find_implicit_links cimport ImplicitLinkFinder
from linking.target_data cimport PrepareForQueries
from linking.intent_data cimport GetImpreciseComponentIntents
from linking.intent_data cimport GetPreciseComponentIntents
from linking.intents cimport ComponentIntent
from linking.intents cimport Intent
from linking.validation cimport PerformValidation
from linking import fetch_data
from linking import intents as intents_mod
from linking import write_results
from primo.linking.find_explicit_links cimport ExplicitLinkFinder
from primo.linking.find_implicit_links cimport ImplicitLinkFinder
from primo.linking.target_data cimport PrepareForQueries
from primo.linking.intent_data cimport GetImpreciseComponentIntents
from primo.linking.intent_data cimport GetPreciseComponentIntents
from primo.linking.intents cimport ComponentIntent
from primo.linking.intents cimport Intent
from primo.linking.validation cimport PerformValidation
from primo.linking import fetch_data
from primo.linking import intents as intents_mod
from primo.linking import write_results
gflags.DEFINE_boolean('computeattributes', True, 'Compute attributes.')
......
......@@ -15,7 +15,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from linking.intents cimport ComponentIntent
from primo.linking.intents cimport ComponentIntent
cdef void AddPreciseIntent(ComponentIntent intent)
cdef void AddImpreciseIntent(ComponentIntent intent)
......
......@@ -16,17 +16,17 @@
# limitations under the License.
"""Global Intent maps and constants."""
from linking.intents cimport ComponentIntent
from linking.intents cimport Intent
from primo.linking.intents cimport ComponentIntent
from primo.linking.intents cimport Intent
import logging
from linking.target_data import BASE_TYPE
from linking.target_data import CLASS
from linking.target_data import PACKAGE
from linking import ic3_data_pb2
from primo.linking.target_data import BASE_TYPE
from primo.linking.target_data import CLASS
from primo.linking.target_data import PACKAGE
from primo.linking import ic3_data_pb2
include 'linking/constants.pxi'
include 'primo/linking/constants.pxi'
LOGGER = logging.getLogger(__name__)
......
......@@ -15,7 +15,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from linking.components cimport Component
from primo.linking.components cimport Component
cdef IntentFilter MakeIntentFilter(object intent_filter_pb,
Component component)
......
......@@ -16,10 +16,10 @@
# limitations under the License.
"""Class and factory for Intent Filters."""
from linking.components cimport Component
from linking.target_data cimport AddIntentFilterAttributes
from primo.linking.components cimport Component
from primo.linking.target_data cimport AddIntentFilterAttributes
from linking import ic3_data_pb2
from primo.linking import ic3_data_pb2
_ACTION_MAIN = u'android.intent.action.MAIN'
......
......@@ -15,7 +15,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from linking.intents cimport Intent
from primo.linking.intents cimport Intent
cdef void UpdateImpreciseDistribution(Intent intent)
cdef void MakeRandomImprecision(Intent intent)
......@@ -17,12 +17,12 @@
"""Module for recording field imprecision distribution and for introducing
imprecise fields according to a given distribution."""
from linking.intents cimport Intent
from primo.linking.intents cimport Intent
from collections import Counter
import random
include 'linking/constants.pxi'
include 'primo/linking/constants.pxi'
# Explicit Intent imprecision counters.
......
......@@ -15,8 +15,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from linking.applications cimport Application
from linking.components cimport Component
from primo.linking.applications cimport Application
from primo.linking.components cimport Component
cdef class Intent(object):
cdef readonly unicode permission
......
......@@ -16,21 +16,21 @@
# limitations under the License.
"""Class and factory for Intent."""
from linking.applications cimport Application
from linking.components cimport Component
from linking.intent_data cimport AddImpreciseIntent
from linking.intent_data cimport AddPreciseIntent
from linking.target_data cimport GetTargetCountForValue
from primo.linking.applications cimport Application
from primo.linking.components cimport Component
from primo.linking.intent_data cimport AddImpreciseIntent
from primo.linking.intent_data cimport AddPreciseIntent
from primo.linking.target_data cimport GetTargetCountForValue
from collections import Counter
from linking import ic3_data_pb2
from linking.target_data import CLASS
from linking.target_data import KIND
from linking.target_data import PACKAGE
from linking.target_data import USED_PERMISSIONS
from primo.linking import ic3_data_pb2
from primo.linking.target_data import CLASS
from primo.linking.target_data import KIND
from primo.linking.target_data import PACKAGE
from primo.linking.target_data import USED_PERMISSIONS
include 'linking/constants.pxi'
include 'primo/linking/constants.pxi'
cdef dict _INTENT_COUNTERS = {ic3_data_pb2.ACTION: Counter(),
......
......@@ -34,7 +34,7 @@ from matplotlib import rcParams
import bloscpack as bp
import numpy as np
from linking.write_results cimport Row
from primo.linking.write_results cimport Row
FONT_SIZE = 22
......
......@@ -15,9 +15,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from linking.applications cimport Application
from linking.components cimport Component
from linking.intent_filters cimport IntentFilter
from primo.linking.applications cimport Application
from primo.linking.components cimport Component
from primo.linking.intent_filters cimport IntentFilter
cdef void PrepareForQueries(set applications)
cdef int GetExportedComponentCount(int kind, set search_space=?)
......
......@@ -23,10 +23,10 @@ DEF DYNAMIC_RECEIVER = 3
from collections import Counter
import logging
from linking.attribute_matching cimport AttributeMap
from primo.linking.attribute_matching cimport AttributeMap
from linking.util import Powerset
from linking import ic3_data_pb2
from primo.linking.util import Powerset
from primo.linking import ic3_data_pb2
LOGGER = logging.getLogger(__name__)
......
......@@ -20,20 +20,20 @@ Goodman-Kruskal's gamma."""
cimport cython
cimport numpy as np
from linking.components cimport GetSkippedFilterCount
from linking.intent_data cimport AddPreciseIntent
from linking.intent_data cimport AddImpreciseIntent
from linking.intent_imprecisions cimport MakeRandomImprecision
from linking.intent_imprecisions cimport UpdateImpreciseDistribution
from linking.intents cimport ComponentIntent
from primo.linking.components cimport GetSkippedFilterCount
from primo.linking.intent_data cimport AddPreciseIntent
from primo.linking.intent_data cimport AddImpreciseIntent
from primo.linking.intent_imprecisions cimport MakeRandomImprecision
from primo.linking.intent_imprecisions cimport UpdateImpreciseDistribution
from primo.linking.intents cimport ComponentIntent
import logging
import numpy as np
import gflags
from linking import intent_data
import linking.find_links
from primo.linking import intent_data
import primo.linking.find_links
FLAGS = gflags.FLAGS
......@@ -72,7 +72,7 @@ cdef void PerformValidation(list intents, bint skip_empty, set components,
# The ground truth contains all links with "full confidence" (priority = 100).
# Any link not in this set has priority 0.
ground_truth, _, _, _, _, _, _ = linking.find_links.FindLinksForIntents(
ground_truth, _, _, _, _, _, _ = primo.linking.find_links.FindLinksForIntents(
precise, set(), skip_empty, components, intent_filters, False, False)
# Store the ground truth targets into a set for efficient lookup.
......@@ -135,7 +135,7 @@ cdef void PerformValidation(list intents, bint skip_empty, set components,
still_precise += 1
validation = new_validation
intent_links, _, _, _, _, _, _ = linking.find_links.FindLinksForIntents(
intent_links, _, _, _, _, _, _ = primo.linking.find_links.FindLinksForIntents(
training, validation, skip_empty, components, intent_filters, True,
True)
......
......@@ -22,8 +22,8 @@ cimport numpy as np
import bloscpack
import numpy as np
from linking.intents cimport ComponentIntent
from linking.intents cimport Intent
from primo.linking.intents cimport ComponentIntent
from primo.linking.intents cimport Intent
DTYPE = [('intent', 'int32'),
......
......@@ -28,10 +28,10 @@ import unittest
sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)),
'..'))
from linking import write_results
from linking.applications import Application
from linking.intents import ComponentIntent
from linking.intents import Intent
from primo.linking import write_results
from primo.linking.applications import Application
from primo.linking.intents import ComponentIntent
from primo.linking.intents import Intent
class MockTarget(object):
......
......@@ -22,36 +22,38 @@ from distutils.core import setup
from distutils.extension import Extension
SRC_DIR = 'linking'
SRC_DIR = 'primo/linking'
if glob.glob(os.path.join(SRC_DIR, '*.pyx')):
try:
from Cython.Distutils import build_ext
use_cython = True
#from Cython.Compiler import Options
#Options.annotate = True
# Options.directive_defaults['profile'] = True
except ImportError:
if glob.glob(os.path.join(SRC_DIR, '*.c')):
use_cython = False
elif glob.glob(os.path.join(SRC_DIR, '*.pyx')):
try:
from Cython.Distutils import build_ext
use_cython = True
#from Cython.Compiler import Options
#Options.annotate = True
# Options.directive_defaults['profile'] = True
except ImportError:
use_cython = False
else:
# We don't ship .pyx files with a source distribution so that users don't have
# to cythonize everything again.
# See http://docs.cython.org/src/reference/compilation.html#distributing-cython-modules.
use_cython = False
# We don't ship .pyx files with a source distribution so that users don't have
# to cythonize everything again.
# See http://docs.cython.org/src/reference/compilation.html#distributing-cython-modules.
use_cython = False
def ScanDir(directory, file_extension, files=[]):
"""Scans the 'linking' directory for extension files, converting
them to extension names in dotted notation."""
"""Scans the 'linking' directory for extension files, converting
them to extension names in dotted notation."""
for file_name in os.listdir(directory):
file_path = os.path.join(directory, file_name)
if os.path.isfile(file_path) and file_path.endswith(file_extension):
files.append(file_path.replace(os.path.sep, '.')[:-len(file_extension)])
elif os.path.isdir(file_path):
ScanDir(file_path, file_extension, files)
return files
for file_name in os.listdir(directory):
file_path = os.path.join(directory, file_name)
if os.path.isfile(file_path) and file_path.endswith(file_extension):
files.append(file_path.replace(os.path.sep, '.')[:-len(file_extension)])
elif os.path.isdir(file_path):
ScanDir(file_path, file_extension, files)
return files
NAME = 'primo'
VERSION = '0.1.0'
......@@ -63,48 +65,53 @@ EMAIL = 'octeau@cse.psu.edu'
LICENSE = 'Apache 2.0'