Changeset 965

Timestamp:

02/22/12 16:41:28 (3 months ago)

Author:

frederick

Message:

Split some of the settings into separate input and output configuration files and updated the source tree accordingly to reflect the new locations of the relocated settings.

Location:

trunk/synthesis/src

Files:

• conf/inputConfiguration.py (added)
• conf/outputConfiguration.py (modified) (5 diffs)
• conf/settings.py (modified) (6 diffs)
• controllers/docs.py (modified) (12 diffs)
• emailprocessor.py (modified) (3 diffs)
• filerouter.py (modified) (2 diffs)
• resttransport.py (modified) (9 diffs)
• selector.py (modified) (4 diffs)
• soaptransport.py (modified) (4 diffs)

trunk/synthesis/src/conf/outputConfiguration.py

@@ -1 +1 @@
 '''see ./docs/output_configurations.readme for information on setting these configurations'''
 #!/usr/bin/env python
-
+import os
 import settings
 
-# Adding a vendor.  Simply append a record by following the format below.
+PROCESSEDFILES_PATH = os.path.join(settings.BASE_PATH, "processed_files")
+if not os.path.exists(PROCESSEDFILES_PATH):
+    os.mkdir(PROCESSEDFILES_PATH)
+
+# Adding a vendor.  Simply append a record by following the sample format below.
 
 Configuration = \
 {
+    # Sample Format
+    'Source ID (e.g. 001)':
+        {
+            'vendor': '',
+            'outputFormat': '', # Possible values: 
+            'destinationURL': '',
+            'transportConfiguration': 'save', # Possible values: "save" (to directory), "soap", "rest", "ftps", "sftp"
+            'encryption': 'none', # Possible values: "none", "3des", "openpgp"
+            # when transportConfiguration is set to "save", set destination to the absolute path of the save directory
+            'destination': settings.BASE_PATH + '/output_files',
+            # frequency controls how often nodebuilder runs and can be set to "asap" (later on maybe "daily", "weekly", monthly")
+            'frequency': 'asap'
+        },
     'occtest':#For HUD files in test_files folder
         {
@@ -14 +31 @@
             'destinationURL': 'https://pix.penguix.net:8023/docs',
             #'destinationURL': 'http://127.0.0.1:5000/docs',
-            # transport type can be set to "save" (to directory), "soap", "rest", "ftps", "sftp"
             'transportConfiguration': 'save',
-            # encryption type can be set to "none", "openpgp", "3des"
             'encryption': '3des',
-            # when transportConfiguration is set to "save", set destination to the absolute path of the save directory
             'destination': settings.BASE_PATH + '/occ_output_files',
-            # frequency controls how often nodebuilder runs and can be set to "asap" (later on maybe "daily", "weekly", monthly")
             'frequency': 'asap'
         },
@@ -28 +41 @@
             'outputFormat': 'pseudo',
             'destinationURL': 'https://pix.penguix.net:8024/docs',
-            #'destinationURL': 'http://127.0.0.1:5001/docs',
-            # transport type can be set to "save" (to directory), "soap", "rest", "ftps", "sftp"
             'transportConfiguration': 'save',
-            # encryption type can be set to "none", "openpgp", "3des"
             'encryption': 'none',
-            # when transportConfiguration is set to "save", set destination to the absolute path of the save directory
             'destination': settings.BASE_PATH + '/tbc_output_files',
-            # frequency controls how often nodebuilder runs and can be set to "asap" (later on maybe "daily", "weekly", monthly")
             'frequency': 'asap'
         },
@@ -42 +50 @@
             'vendor': 'Orange County Corrections',
             'outputFormat': 'svcpoint5',
-            #'destinationURL': 'https://pix.penguix.net:8024/docs',
             'destinationURL': 'http://fby.homeip.net:8092/docs',
-            #'destinationURL': 'http://127.0.0.1:5000/docs',
-            # transport type can be set to "save" (to directory), "soap", "rest", "ftps", "sftp"
             'transportConfiguration': 'save',
-            # encryption type can be set to "none", "openpgp", "3des"
             'encryption': 'none',
-            # when transportConfiguration is set to "save", set destination to the absolute path of the save directory
             'destination': settings.BASE_PATH + '/output_files',
-            # frequency controls how often nodebuilder runs and can be set to "asap" (later on maybe "daily", "weekly", monthly")
             'frequency': 'asap'
         },
@@ -130 +132 @@
         },
 }
-

trunk/synthesis/src/conf/settings.py

@@ -17 +17 @@
 GUI = False  
 # DB settings:
-DB_DATABASE = "your_database_name_here(synthesis perhaps)"
-DB_USER = "your_database_user_name_here"
-DB_PASSWD = "your_password_here"
+DB_DATABASE = "synthesis"
+DB_USER = "synthesis"
+DB_PASSWD = "synthesis"
 DB_PORT = 5432
 DB_HOST = "localhost"
@@ -29 +29 @@
 #SOURCE_PATH = 'synthesis'
 #BASE_PATH = os.path.join(os.getcwd(), SOURCE_PATH)
-BASE_PATH = "/home/username/myrestservice/synthesis/synthesis"
+BASE_PATH = "~/myrestservice/synthesis/synthesis"
 print "BASE_PATH is: ", BASE_PATH
 #import pydevd; pydevd.settrace()
 #ABS_SOURCE_PATH = os.path.abspath(os.getcwd() + SOURCE_PATH)
 PATH_TO_GPG = '/usr/bin/gpg'
-PGPHOMEDIR = ''
+PGPHOMEDIR = '~/.gnupg'
 PASSPHRASE = ''
 FINGERPRINT = ''
+DES3_KEY = ''
+
 JFCS_SOURCE_ID = 734
 JFCS_AGENCY_ID = 711
@@ -46 +48 @@
 NUMBER_OF_PROCESSES = 1 
 
-# <rim:Classification /> properties
-AUTHOR_PERSON = "John Doe"
-AUTHOR_INSTITUTION = "ACME"
-AUTHOR_ROLE = "Wise Man"
-AUTHOR_SPECIALTY = "Mediation"
-NODE_REPRESENTATION = "Not Available"
-CODING_SCHEME = "LOINC"
-LOCALIZED_STRING = "Not Available"
+# <rim:Classification /> properties (used by the soaptransport)
+AUTHOR_PERSON = ""
+AUTHOR_INSTITUTION = ""
+AUTHOR_ROLE = ""
+AUTHOR_SPECIALTY = ""
+NODE_REPRESENTATION = ""
+CODING_SCHEME = ""
+LOCALIZED_STRING = ""
 
-# Input files Processing path
-# New 'Customers' input file path must be put into this 'list'.  Add it to the list mechanism.  
-INPUTFILES_PATH = [
-            BASE_PATH + "/input_files"
-            ,
-            ]
-WEB_SERVICE_INPUTFILES_PATH = [
-            BASE_PATH + "/ws_input_files"
-            ,
-            ]
 # subfolder 
 #XSD_PATH = SOURCE_PATH + "/" + "xsd"
@@ -73 +65 @@
 #    os.mkdir(OUTPUTFILES_PATH)
     
-USEDFILES_PATH = os.path.join(BASE_PATH, "used_files")
-if not os.path.exists(USEDFILES_PATH):
-    os.mkdir(USEDFILES_PATH)
-    
-FAILEDFILES_PATH = os.path.join(BASE_PATH, "failed_files")
-if not os.path.exists(FAILEDFILES_PATH):
-    os.mkdir(FAILEDFILES_PATH)
-
-PROCESSEDFILES_PATH = os.path.join(BASE_PATH, "processed_files")
-if not os.path.exists(PROCESSEDFILES_PATH):
-    os.mkdir(PROCESSEDFILES_PATH)    
-
 #logging settings file location setup
 logging_level = 0
@@ -98 +78 @@
 if not os.path.exists(LOGS):
     os.mkdir(LOGS)
-
-PROCESSED_PATH = ""
 
 SCHEMA_DOCS = {
@@ -127 +105 @@
 SMTPSENDERPWD = 'mysecret'
 
-# SMTP Mail recipients is a dictionary that must be defined for each source of input files
-SMTPRECIPIENTS = {    
-        "~/myrestservice/synthesis/synthesis/input_files":
-        {
-        'VENDOR_NAME': 'SomeVendor',
-        'SMTPTOADDRESS': ['someone@somedomain.com',],
-        'SMTPTOADDRESSCC': [],
-        'SMTPTOADDRESSBCC': [],
-        'FINGERPRINT':'',
-        'USES_ENCRYPTION':False
-        }
-    ,"~/myrestservice/synthesis/synthesis/input_files2":
-        {
-        'VENDOR_NAME': 'SomeVendor2',
-        'SMTPTOADDRESS': ['admin@superhost.com',],
-        'SMTPTOADDRESSCC': [],
-        'SMTPTOADDRESSBCC': [],
-        'FINGERPRINT':'',
-        'USES_ENCRYPTION':False
-        }
-    ,"~/myrestservice/synthesis/synthesis/input_files3":
-        {
-        'VENDOR_NAME': '',
-        'SMTPTOADDRESS': ['sammy.davis@jr.com',],
-        'SMTPTOADDRESSCC': [],
-        'SMTPTOADDRESSBCC': [],
-        'FINGERPRINT':'',
-        'USES_ENCRYPTION':False
-        }
-# output processing
-    ,"~/myrestservice/synthesis/synthesis/output_files":
-        {
-        'VENDOR_NAME': '',
-        'SMTPTOADDRESS': ['user@host.com',],
-        'SMTPTOADDRESSCC': [],
-        'SMTPTOADDRESSBCC': [],
-        'FINGERPRINT':'',
-        'USES_ENCRYPTION':False
-        }
-    ,"~/myrestservice/synthesis/synthesis/output_files2":
-        {
-        'VENDOR_NAME': '',
-        'SMTPTOADDRESS': ['admin@somehost.com',],
-        'SMTPTOADDRESSCC': [],
-        'SMTPTOADDRESSBCC': [],
-        'FINGERPRINT':'',
-        'USES_ENCRYPTION':False
-        }
-    }
 try:
     from local_settings import *
 except ImportError:
     pass
-
-

trunk/synthesis/src/controllers/docs.py

@@ -1 @@
-#use_encryption = True
-use_encryption = False
-
+from conf import settings
+from conf import inputConfiguration
 import os
 import logging
@@ -11 +10 @@
 import urllib
 from webob import Request
-if use_encryption:
+if inputConfiguration.USE_ENCRYPTION:
     from Encryption import *
-from conf import settings
 from selector import Selector
 import base64
@@ -59 +57 @@
         file_suffix_enc = '_encrypted.xml'
         file_suffix_unenc = '_unencrypted.xml'
-        if use_encryption:
+        if inputConfiguration.USE_ENCRYPTION:
             print "using encryption"
             file_name = file_prefix + file_suffix_enc
-        elif not use_encryption:
+        elif not inputConfiguration.USE_ENCRYPTION:
             print "not using encryption"
             file_name = file_prefix + file_suffix_unenc
@@ -68 +66 @@
             print "not sure if using encrypted file or not"
         
-        if not os.path.exists(settings.WEB_SERVICE_INPUTFILES_PATH[0]):
-                os.mkdir(settings.WEB_SERVICE_INPUTFILES_PATH[0])
-        file_full_path = os.path.join(settings.WEB_SERVICE_INPUTFILES_PATH[0], file_name)
+        if not os.path.exists(inputConfiguration.WEB_SERVICE_INPUTFILES_PATH[0]):
+                os.mkdir(inputConfiguration.WEB_SERVICE_INPUTFILES_PATH[0])
+        file_full_path = os.path.join(inputConfiguration.WEB_SERVICE_INPUTFILES_PATH[0], file_name)
         print 'file_full_path: ', file_full_path
         
         #open file 
-        if use_encryption:
+        if inputConfiguration.USE_ENCRYPTION:
             try:
                 print "trying to open encrypted file"
@@ -80 +78 @@
             except:
                 print "Error opening encrypted instance file for writing"
-        if not use_encryption:
+        if not inputConfiguration.USE_ENCRYPTION:
             try:
                 print "trying to open unencrypted file"
@@ -88 +86 @@
                 
         #write to file
-        if use_encryption:
+        if inputConfiguration.USE_ENCRYPTION:
             print 'writing', file_name, 'to', server_root, 'for decryption'
             print 'encrypted_file is', encrypted_file
@@ -94 +92 @@
             encrypted_file.close()
             
-        if not use_encryption:
+        if not inputConfiguration.USE_ENCRYPTION:
             print 'writing', file_name, 'to', server_root, 'server root for parsing'
             print 'unencrypted_file is', unencrypted_file
@@ -109 +107 @@
         #assume file is encrypted, since it can be difficult to tell if it is.  We could look for XML structures, but how do you easily tell bad/invalid  XML apart from encrypted?  If not encrypted, that's a problem.
         #decrypt file
-        if use_encryption:
+        if inputConfiguration.USE_ENCRYPTION:
             try:
                 encrypted_file = open(file_full_path, 'r') 
@@ -148 +146 @@
                 file_suffix_unenc = '_decrypted.xml'
                 file_name = file_prefix + file_suffix_unenc
-                file_full_path =  settings.INPUTFILES_PATH[0] + '/' + file_name
+                file_full_path =  inputConfiguration.INPUTFILES_PATH[0] + '/' + file_name
                 try:
                     decrypted_file = open(file_full_path, 'w')
@@ -167 +165 @@
         
         #read in candidate XML file
-#        if use_encryption:
+#        if inputConfiguration.USE_ENCRYPTION:
 #            try:
 #                unencrypted_file = open(file_full_path, 'r') 
@@ -173 +171 @@
 #                print "couldn't open decrypted file for reading"
 #            
-#        if not use_encryption:
+#        if inputConfiguration.USE_ENCRYPTION:
 #            try:
 #                unencrypted_file = open(file_full_path, 'r') 
@@ -193 +191 @@
             print "moving valid file ", file_name, "over to input_files for shredding"
             import fileutils
-            fileutils.moveFile(file_full_path, settings.INPUTFILES_PATH[0])
+            fileutils.moveFile(file_full_path, inputConfiguration.INPUTFILES_PATH[0])
             return message
         except:

trunk/synthesis/src/emailprocessor.py

@@ -1 +1 @@
 from smtplibrary import smtpInterface
 from conf import settings
+from conf import inputConfiguration
 import os
 from security import Security
@@ -15 +16 @@
         if docName <> '':
             folderName = os.path.split(docName)[0]
-            self.mailSystem.setRecipients(settings.SMTPRECIPIENTS[folderName])
+            self.mailSystem.setRecipients(inputConfiguration.SMTPRECIPIENTS[folderName])
             self.docName = docName
         else:
@@ -22 +23 @@
                 try:
                     folderName = os.path.split(docs[0])[0]
-                    self.mailSystem.setRecipients(settings.SMTPRECIPIENTS[folderName])
+                    self.mailSystem.setRecipients(inputConfiguration.SMTPRECIPIENTS[folderName])
                 except KeyError:
                     raise

trunk/synthesis/src/filerouter.py

@@ -1 +1 @@
 import fileutils
+from conf import inputConfiguration
 from conf import settings
 
@@ -9 +10 @@
     def moveUsed(self, file_name):
         if settings.DEBUG:
-            print "moving ", file_name, "to", settings.USEDFILES_PATH
-        fileutils.moveFile(file_name, settings.USEDFILES_PATH)
+            print "moving ", file_name, "to", inputConfiguration.USEDFILES_PATH
+        fileutils.moveFile(file_name, inputConfiguration.USEDFILES_PATH)
 
     def moveFailed(self, fileName):
-        fileutils.moveFile(fileName, settings.FAILEDFILES_PATH)
+        fileutils.moveFile(fileName, inputConfiguration.FAILEDFILES_PATH)
 
 

trunk/synthesis/src/resttransport.py

@@ -3 +3 @@
 HAS_CONF = True
 HAS_ENCRYPTION = True
-#POST_URL = "http://127.0.0.1:5000/docs"
-OCC_POST_URL = "https://pix.penguix.net:8023/occdocs"
-TBC_POST_URL = "https://pix.penguix.net:8023/tbcdocs"
+SENT_TO_LOCAL = True
+
+if SENT_TO_LOCAL:
+# LOCALHOST URLS
+    OCC_POST_URL = "http://127.0.0.1:5000/docs"
+    TBC_POST_URL = "http://127.0.0.1:5000/docs"
+else:
+# TEST SERVER URLS
+    OCC_POST_URL = "https://pix.penguix.net:8023/docs"
+    TBC_POST_URL = "https://pix.penguix.net:8024/docs"
+
 import base64
 try:
@@ -72 +80 @@
     <ext:Source>
         <ext:SourceID>
-                <hmis:IDStr>occtest</hmis:IDStr>
+                <hmis:IDStr>003</hmis:IDStr>
         </ext:SourceID>
         <ext:SoftwareVendor>OCC</ext:SoftwareVendor>
@@ -147 +155 @@
     
     # test results
-    #print "Result of OCC test (unencrypted): ", rest.post("occtest", occ_xml)
-    
     if HAS_ENCRYPTION:
         des3 = DES3()
@@ -154 +160 @@
         #print encrypted_data
         print "Result of OCC test (encrypted): ", rest.post("occtest", encrypted_data, use_base64=True)
+    else:
+        print "Result of OCC test (unencrypted): ", rest.post("occtest", occ_xml, use_base64=False)
 
 def tbctest():
@@ -164 +172 @@
     xmlns:hmis="http://www.hmis.info/schema/3_0/HUD_HMIS.xsd" 
     xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" 
-    xsi:schemaLocation="http://xsd.alexandriaconsulting.com/repos/trunk/HUD_HMIS_XML/TBC_Extend_HUD_HMIS.xsd http://xsd.alexandriaconsulting.com/repos/trunk/HUD_HMIS_XML/TBC_Extend_HUD_HMIS.xsd"
+    xsi:schemaLocation="http://xsd.alexandriaconsulting.com/repos/trunk/HUD_HMIS_XML/TBC_Extend_HUD_HMIS.xsd http://xsd.alexandriaconsulting.com/repos/trunk/HUD_HMIS_XML/TBC_Extend_HUD_HMIS.xsd" 
     ext:version="3.0">
     <ext:Source>
         <ext:SourceID >
-                <hmis:IDStr>tbctest</hmis:IDStr>
+                <hmis:IDStr>003</hmis:IDStr>
         </ext:SourceID>
-        <ext:SoftwareVendor>TBC</ext:SoftwareVendor>
+        <ext:SoftwareVendor>Vendor Name Here</ext:SoftwareVendor>
         <ext:SoftwareVersion>7.1</ext:SoftwareVersion>
         <ext:SourceName>Source Name Here</ext:SourceName>
@@ -176 +184 @@
             <ext:ExportID >
                 <!-- Since this is the first export, it's "1".  The second will be "2". -->
-                    <hmis:IDNum>11</hmis:IDNum>
+                    <hmis:IDNum>1010111</hmis:IDNum>
             </ext:ExportID>
             <ext:ExportDate>2011-09-23T18:51:58</ext:ExportDate>
@@ -186 +194 @@
             <ext:Person>
                 <ext:PersonID>
-                    <hmis:IDNum>090888539</hmis:IDNum>
+                    <hmis:IDNum>2090888539</hmis:IDNum>
                 </ext:PersonID>
                 <ext:DateOfBirth>
@@ -297 +305 @@
     
     # test results
-    #print "Result of TBC test (unencrypted): ", rest.post("tbctest", tbc_xml)
-
     if HAS_ENCRYPTION:
         gpg = GPG()
@@ -304 +310 @@
         #print encrypted_data
         print "Result of TBC test (encrypted): ", rest.post("tbctest", encrypted_data, use_base64=True)
+    else:
+        print "Result of TBC test (unencrypted): ", rest.post("tbctest", tbc_xml, use_base64=False)
 
 if __name__ == "__main__":

trunk/synthesis/src/selector.py

@@ -16 +16 @@
 import Queue
 from conf import settings
+from conf import inputConfiguration
 from synthesis.emailprocessor import XMLProcessorNotifier
 from filerouter import Router
@@ -37 +38 @@
 
         #change this so that it gets the watch_dir from the .ini file
-        dir_to_watch = settings.INPUTFILES_PATH 
+        dir_to_watch = inputConfiguration.INPUTFILES_PATH 
         self.queue = Queue.Queue(0)
         self.file_input_watcher = FileInputWatcher(dir_to_watch, self.queue)
@@ -79 +80 @@
         if os.path.isdir(folderName): 
             try:
-                self.ProcessingOptions = settings.SMTPRECIPIENTS[folderName]
+                self.ProcessingOptions = inputConfiguration.SMTPRECIPIENTS[folderName]
             except:
                 raise
@@ -162 +163 @@
         listOfFiles = list()
         # Loop over list of file locations [list]
-        for folder in settings.INPUTFILES_PATH:
+        for folder in inputConfiguration.INPUTFILES_PATH:
             listOfFiles.extend(fileutils.grabFiles(path.join(folder,'*')))
             if settings.DEBUG:

trunk/synthesis/src/soaptransport.py

@@ -7 +7 @@
 import urlparse
 import uuid
+import base64
 
 class SoapEnv():
@@ -134 +135 @@
     def send_soap_envelope(self, ccd_data, test=False):
         # construct the message and header
-        self._transport_properties["CCD"] = ccd_data
+        self._transport_properties["CCD"] = base64.b64encode(ccd_data)
         soap_env = self._ENVELOPE_TEMPLATE % self._transport_properties
         headers = {
@@ -152 +153 @@
             import pprint as pp
             pp.pprint(headers)
-            print soap_env
+            #print soap_env
+            fo = open('soap_envelope.txt', 'w')
+            fo.write(soap_env)
+            fo.flush()
+            fo.close()
             return (True, "True")
 
@@ -337 +342 @@
 
     soap = SoapEnv('iH9HiPbW40JbS5m_')
-    soap.send_soap_envelope(ccd_data)
+    soap.send_soap_envelope(ccd_data, True)