Merge pull request 'Add S3 functionality' (#2) from s3-connect into master

Reviewed-on: tyler/Meme-service#2

.gitignore

@@ -181,3 +181,4 @@ tags
 # Persistent undo
 [._]*.un~
 
+apply_environment.sh

src/api/search.py

@@ -1,18 +1,39 @@
 from flask_restx import Namespace, Resource, fields
+from flask_restx import reqparse
+import s3Client
+import logging
 
+# Exported namespace
 api = Namespace('search', description='Searching for memes')
 
-query = api.model('Query', {
+def getClientSafely():
-    'query': fields.String(required=True, description='Search string', example='doge'),
+    logging.debug("Getting a client safely...")
+    client = None
+    try:
+        client = s3Client.getClient()
+    except Exception as e:
+        logging.critical("Failed to retrive a client : " + str(e))
 
-    })
+    return client
 
-@api.route('/exact')
+
+@api.route('/exact/<string:query>')
 @api.doc(params={
     'query': 'Name of the meme you are looking for'
     })
 class exactSearch(Resource):
     @api.doc('search')
-    def get(self):
+    @api.response(200, 'Sucess')
-        return {"message" : "hello world!"}
+    @api.response(500, 'S3 Error')
+    def get(self, query):
+        logging.debug("Getting a client safely...")
+        client = getClientSafely()
+        if client is None:
+            return {
+                    "message": "Error connecting to S3"
+                }, 500
+        if query in client.getCurrentMemeList():
+            return "nice"
+        else:
+            return "boo"
 

src/app.py

@@ -4,6 +4,9 @@
 from flask import Flask
 from api import api
 import os
+import logging
+
+logging.basicConfig(level=logging.INFO)
 
 isDebug = True
 

src/s3Client.py

@@ -0,0 +1,105 @@
+import logging
+import os
+
+
+
+from minio import Minio
+from minio.commonconfig import Tags
+from minio.error import S3Error
+from datetime import datetime
+
+
+S3_URL = ""
+S3_UN  = ""
+S3_PW  = ""
+S3_TLS = True
+S3_BUCKET = ""
+
+gclient = None
+
+def getClient():
+    global gclient
+    global S3_URL
+    global S3_UN
+    global S3_PW
+    global S3_TLS
+    global S3_BUCKET
+    if gclient != None:
+        return gclient
+
+    if "S3_URL" not in os.environ:
+        raise Exception("S3_URL is not set!")
+    S3_URL = os.environ["S3_URL"]
+    logging.info("Using S3_URL : " + S3_URL )
+
+
+    if "S3_UN" not in os.environ:
+        raise Exception("S3_UN is not set!")
+    S3_UN = os.environ["S3_UN"]
+    logging.info("Using S3_UN : " + S3_UN)
+
+    if "S3_PW" not in os.environ:
+        raise Exception("S3_PW is not set!")
+    S3_PW = os.environ["S3_PW"]
+    logging.info("Using S3_PW : " + S3_PW)
+
+    if "S3_BUCKET" not in os.environ:
+        raise Exception("S3_BUCKET is not set!")
+    S3_BUCKET = os.environ["S3_BUCKET"]
+    logging.info("Using S3_BUCKET : " + S3_BUCKET)
+
+    # override defaults
+    if "S3_TLS" in os.environ:
+        S3_TLS = os.environ["S3_TLS"].lower() in ("yes", "true", "1", "t")
+    logging.info("Using S3_TLS : " + str(S3_TLS))
+
+    client = Minio(S3_URL,
+                   access_key=S3_UN,
+                   secret_key=S3_PW,
+                   secure=S3_TLS)
+    found = client.bucket_exists(S3_BUCKET)
+    if not found:
+        client.make_bucket(S3_BUCKET)
+        logging.info(f"Failed to find bucket " + S3_BUCKET + " so I made it instead")
+    else:
+        logging.info(f"Found bucket " + S3_BUCKET)
+
+    clientObject = Client(client)
+    gclient = clientObject
+
+    return gclient
+
+class Client:
+    allMemes = set()
+    memesToMd5 = dict()
+    lastCheckedAllMemes = datetime.strptime("2000-01-01 01:01:01", "%Y-%m-%d %H:%M:%S")
+    client = None
+
+    def __init__(self, client):
+        global gclient
+        if gclient is not None:
+            raise Exception("Object already exists! use getClient")
+        logging.debug("Making a new client")
+        if client is not None:
+            self.client = client
+        else:
+            raise Exception("Improper object passed for client!")
+
+    def getCurrentMemeList(self, force=False):
+        now = datetime.now()
+        if (now - self.lastCheckedAllMemes).seconds > 300 or force:
+            logging.info("Enough time has elapsed, refreshing meme cache...")
+            self.lastCheckedAllMemes = now
+
+            self.allMemes.clear()
+            self.memesToMd5.clear()
+
+            for obj in self.client.list_objects(S3_BUCKET):
+                if not obj.is_dir:
+                    self.allMemes.add(obj.object_name)
+                    self.memesToMd5[obj.etag] = obj.object_name
+            logging.info("Finished fetching " + str(len(self.allMemes)) + " memes")
+
+        return self.allMemes
+
+