Initial commit

Author: Eric Woolard

README.md

@@ -0,0 +1,18 @@
+New-Modmail Fetcher
+------------
+This was created using the bare Reddit API for new-modmail, since at the time PRAW had yet to update.
+The new-modmail API is pretty messy, so I did the best I could with what I gots. 
+
+INFO
+----------
+It currently runs every 2 minutes, checks the most recent 10 modmails in the 'Archived' section of modmail,
+and writes the response to a cached json file. It then goes through the cached copy, grabs info for each 
+conversation and logs a new document in the modmail.new collection in mongo. Since individual message replies
+to a particular modmail conversation are not accessible from the initial request, it also makes a separate
+request to the `/conversations/{ConversationID}` endpoint to retrieve individual responses for each modmail
+conversation stored in the cached response. If it finds a conversation by ID that already exists in the 
+mongodb collection, it updates it with any new messages.
+
+One downside to the new-modmail API is that the response does not include a key for a created_at value. 
+Instead, we're given times for `lastUpdated`, `lastUserUpdate` and `lastModUpdate`. This script uses 
+`lastUpdated`, so if a modmail conversation gets a new message, that timestamp also gets updated. 

cfg.py

@@ -0,0 +1,79 @@
+# System imports
+from time import time
+# Third-party imports
+import praw
+# Project imports
+import file_manager
+
+r = None
+config_path = 'config/'  # Root of all configuration files
+last_updated = 0
+cached_returns = {}  # Used to, well, cache function returns. Not wildly
+
+
+# used in the configuration files, but a handy
+# option nonetheless.
+# TODO: Look into streamlining the caching process
+
+# Aliases of the file_manager functions to control the relative path
+def read(relative_path):
+    global config_path
+    return file_manager.read(config_path + relative_path)
+
+
+def readJson(relative_path):
+    global config_path
+    return file_manager.readJson(config_path + relative_path)
+
+
+def save(relative_path, data):
+    global config_path
+    return file_manager.save(config_path + relative_path, data)
+
+
+def saveJson(relative_path, data):
+    global config_path
+    return file_manager.saveJson(config_path + relative_path, data)
+
+
+# Bot settings
+def getSettings():
+    import authentication
+    global cached_returns
+    global last_updated
+    global config_path
+    global r
+    global profile
+
+    # Only allow updating the settings once every couple minutes
+    if 'settings' in cached_returns and time() - cached_returns['settings']['updated'] < 60 * 2:
+        return cached_returns['settings']['return']
+
+    settings = readJson('settings.json')
+
+    # Get the OAuth information
+    settings['bot'] = getAccounts()
+
+    if 'settings' not in cached_returns:
+        cached_returns['settings'] = {}
+    cached_returns['settings']['updated'] = time()
+    cached_returns['settings']['return'] = settings
+
+    return settings
+
+
+# Bot account OAuth information // These don't follow the profile,
+# keep all bots in the same accounts.json file
+def getAccounts():
+    global config_path
+    return file_manager.readJson(config_path + 'accounts.json')
+
+
+def setAccounts(newAccounts):
+    global config_path
+    file_manager.saveJson(config_path + 'accounts.json', newAccounts)
+
+
+# The Reddit app OAuth info necessary for registering ourselves as legit
+def getOAuthInfo():
+    return readJson('oauth.json')

config/output_example.json

@@ -0,0 +1,72 @@
+{
+    "conversationIds": [],
+    "viewerId": "",
+    "conversations": {
+        "id": {
+            "lastUpdated": "",
+            "subject": "subject",
+            "authors": [
+                {
+                    "isDeleted": false,
+                    "isOp": true,
+                    "isParticipant": false,
+                    "name": "",
+                    "isHidden": false,
+                    "isMod": true,
+                    "isAdmin": false,
+                    "id": 
+                }
+            ],
+            "state": 2,
+            "isRepliable": true,
+            "isAuto": false,
+            "id": "",
+            "isInternal": false,
+            "owner": {
+                "type": "subreddit",
+                "displayName": "GlobalOffensive",
+                "id": "t5_2sqho"
+            },
+            "objIds": [
+                {
+                    "key": "messages",
+                    "id": ""
+                }
+            ],
+            "lastUnread": null,
+            "numMessages": 1,
+            "isHighlighted": false,
+            "lastUserUpdate": null,
+            "lastModUpdate": "",
+            "participant": {
+                "isDeleted": false,
+                "isOp": false,
+                "isParticipant": true,
+                "name": "",
+                "isHidden": false,
+                "isMod": false,
+                "isAdmin": false,
+                "id": 
+            }
+        }
+    },
+    "messages": {
+        "id": {
+            "body": "",
+            "isInternal": false,
+            "bodyMarkdown": "",
+            "author": {
+                "isDeleted": false,
+                "isOp": false,
+                "isParticipant": false,
+                "name": "",
+                "isHidden": false,
+                "isMod": true,
+                "isAdmin": false,
+                "id": 
+            },
+            "date": "2017-03-26T12:44:21.793602+00:00",
+            "id": ""
+        }
+    }
+}

config/settings.json

@@ -0,0 +1,16 @@
+{
+	"reddit": {
+		"subreddit": "subreddit",
+		"username": "user",
+		"password": "hunter4",
+        "client_id": "client_id",
+        "client_secret": "client_secret"
+	},	
+	"db": {
+		"host": "host",
+		"port": 0000,
+		"username": "user",
+		"password": "hunter5",
+		"database": "dbname"
+	}
+}

file_manager.py

@@ -0,0 +1,112 @@
+# System imports
+import errno
+import io
+import json
+import os
+import sys
+# Project imports
+import log
+
+
+##############################
+#	   BASIC OPERATIONS
+##############################
+
+# Saves string of text to a file
+def save(path, data_str, mode='w'):
+    path = ensureAbsPath(path)
+    try:
+        with io.open(path, mode, encoding='utf-8') as f:
+            f.write(str(data_str))
+    except IOError:
+        # This block of code will run if the path points to a file in a
+        # non-existant directory.  To fix this, create the necessary
+        # director(y/ies) and call this function again to create the file.
+        directory = os.path.dirname(path)
+        if not os.path.isdir(directory):
+            try:
+                os.makedirs(directory)
+            except OSError as oserr:
+                pass
+        if os.path.isdir(directory):
+            return save(path, data_str, mode)
+        else:
+            log.error('Could not write to ' + path)
+            return False
+    else:
+        return True
+
+
+# Reads text from a file as a string and returns it
+def read(path):
+    path = ensureAbsPath(path)
+    try:
+        return open(path, encoding="utf8").read()
+    except IOError:
+        log.error('Could not find or open file: ' + path)
+        return False
+
+
+# Deletes the file at the specified path. Cryptic, I know.
+def delete(path):
+    path = ensureAbsPath(path)
+    try:
+        os.remove(path)
+    except OSError:
+        log.error('Could not remove file: ' + path)
+        return False
+    else:
+        return True
+
+
+# If any relative paths are given, make them relative to the bot's working dir
+def ensureAbsPath(path):
+    botRootDir = os.path.dirname(os.path.abspath(sys.argv[0])) + '/'
+    return path if os.path.isabs(path) else botRootDir + path
+
+
+##############################
+#		  JSON FILES
+##############################
+
+# Converts object to JSON, then saves it to a file
+def saveJson(path, data):
+    return save(
+        path,
+        json.dumps(
+            data,
+            ensure_ascii=False,
+            indent=4,
+            separators=(',', ': ')
+        )
+    )
+
+
+# Reads text from a file, converts it to JSON, and returns it
+def readJson(path):
+    f = read(path)
+    if f:
+        try:
+            return json.loads(f)
+        except ValueError:
+            log.error('Could not parse JSON file: ' + path)
+            return False
+    else:
+        return False
+
+
+##############################
+#	   MISC HELPER FUNCS
+##############################
+
+# Appends data to a file rather than overwriting it -- useful for logging
+def append(path, data):
+    return save(path, data + '\n', 'a')
+
+
+# Grabs the file contents and returns them, then deletes the file
+# This is primarily used for the temporary logging mechanism
+def readAndDelete(path):
+    fileContents = read(path)
+    delete(path)
+    return fileContents