Fix issue with nsfw header not passing correctly

Reviewed-on: tyler/Meme-service#13

.drone.yml

@@ -0,0 +1,28 @@
+kind: pipeline
+name: default
+
+steps:
+  - name: docker
+    image: plugins/docker
+    settings:
+      username:
+        from_secret: dockerhub_username
+      password:
+        from_secret: dockerhub_password
+      repo:
+        from_secret: dockerhub_repo
+      tags: latest
+    when:
+      branch:
+        - master
+  - name: notify
+    image: clortox/drone-ntfy
+    settings:
+      URL: https://ntfy.clortox.com
+      USERNAME: drone
+      PASSWORD:
+        from_secret: ntfy_password
+      TOPIC: drone-builds
+      MESSAGE: Meme API build finished!
+      CLICK: https://drone.clortox.com/Infrastructure/Meme-service
+      TITLE: Meme API build

.gitignore

@@ -182,3 +182,4 @@ tags
 [._]*.un~
 
 apply_environment.sh
+docker-compose.yml

Dockerfile

@@ -0,0 +1,9 @@
+FROM python
+
+WORKDIR /usr/src/app
+
+COPY ./src ./
+
+RUN pip install -r requirements.txt
+
+CMD "python" "./app.py"

README.md

@@ -1,3 +1,18 @@
 # Meme-service
 
+[![Build Status](https://drone.clortox.com/api/badges/Infrastructure/Meme-service/status.svg)](https://drone.clortox.com/Infrastructure/Meme-service)
+
 A meme serving microservice, written using flask_restx
+
+## Environment Variables
+
+| Variable  | Usage                                     |
+| --------  | -----                                     |
+| S3_BUCKET | Bucket name                               |
+| S3_URL    | S3 server URL                             |
+| S3_UN     | S3 Username                               |
+| S3_PW     | S3 Password                               |
+| S3_TLS    | Use TLS? Boolean                          |
+| CACHE_TTL | How long do we cache whats in the bucket? |
+| PORT      | Port to run on inside the container       |
+| DEBUG     | Debug mode? Boolean                       |

src/api/__init__.py

@@ -1,10 +1,14 @@
 from flask_restx import Api
 from .search import api as searchNamespace
+from .get import api as getNamespace
+from .util import api as utilNamespace
 
 api = Api(
-        title='Search',
+        title='Memes',
         version=1.0,
-        description='Searching the collection'
+        description='A programatic interface to my meme collection'
         )
 
 api.add_namespace(searchNamespace)
+api.add_namespace(getNamespace)
+api.add_namespace(utilNamespace)

src/api/clientGetter.py

@@ -0,0 +1,12 @@
+import logging
+import s3Client
+
+def getClientSafely():
+    logging.debug("Getting a client safely...")
+    client = None
+    try:
+        client = s3Client.getClient()
+    except Exception as e:
+        logging.critical("Failed to retrive a client : " + str(e))
+
+    return client

src/api/get.py

@@ -0,0 +1,164 @@
+from flask_restx import Namespace, Resource, fields
+from flask_restx import reqparse
+from flask import make_response, abort, request, Response
+from minio.commonconfig import Tags
+from werkzeug.datastructures import FileStorage
+from api.clientGetter import getClientSafely
+import logging
+import random
+
+# Exported namespace
+api = Namespace('resource', description='Interact with the raw underlying files. This namespace does NOT speak json, just raw files')
+
+uploadFields = {'name' : fields.String(title='Name',
+                                       description='File name of your meme',
+                                       required=True,
+                                       example='Funny.mp4'),
+                'uploader' : fields.String(title='Uploader',
+                                           description='Name of the user who uploaded the meme',
+                                           required=True),
+                'nsfw': fields.Boolean(title='NSFW',
+                                       description='Is this NSFW/Spoilable?',
+                                       default=False),
+                'file': fields.String(title='File',
+                                      description='File as Base64'),
+                }
+
+uploadForm = api.parser()
+uploadForm.add_argument('file',
+                        location='files',
+                        type=FileStorage,
+                        required=True)
+uploadForm.add_argument('name',
+                        location='headers',
+                        type=str,
+                        required=True)
+uploadForm.add_argument('uploader',
+                        location='headers',
+                        type=str,
+                        required=True)
+uploadForm.add_argument('nsfw',
+                        location='headers',
+                        type=str,
+                        required=True)
+
+@api.route('/exact/<string:file_name>')
+@api.route('/<string:file_name>', doc={
+    "description" : "Alias for /exact/{query}"
+    })
+@api.doc(description="Interact with exact raw files.")
+class getExactFile(Resource):
+    @api.doc('get')
+    @api.response(200, 'Sucess')
+    @api.response(500, 'S3 Error')
+    @api.response(404, 'Requested file not found')
+    def get(self, file_name):
+        client = getClientSafely()
+        if client is None:
+            abort(500, "S3 failed to start")
+        if file_name in client.getCurrentMemeList():
+            return make_response(client.getMeme(file_name))
+        else:
+            abort(400, "Requested file '" + file_name + "' not found")
+
+@api.route('/')
+class addFile(Resource):
+    @api.response(200, 'Sucess')
+    @api.response(500, 'S3 Error')
+    @api.response(400, 'Bad request')
+    @api.expect(uploadForm)
+    def post(self):
+        client = getClientSafely()
+        if client is None:
+            abort(500, "S3 failed to start")
+
+        args     = uploadForm.parse_args()
+        file     = args['file']
+        fileName = args['name']
+        uploader = args['uploader']
+        nsfw     = args['nsfw']
+
+        print(args)
+        print(nsfw)
+        print(str(nsfw))
+
+        tags = Tags.new_object_tags()
+        tags["uploader"] = uploader
+        tags["nsfw"] = str(nsfw)
+
+        if client.addMeme(fileContents=file,
+                          name=fileName,
+                          tags=tags):
+            return {"message" : "success", "sucess" : True}
+        else:
+            return {"message" : "failure", "success" : False}, 500
+
+
+
+@api.route('/random')
+@api.doc(description="Returns a random meme")
+class getRandomFile(Resource):
+    @api.doc('get')
+    @api.response(200, 'Sucess')
+    @api.response(500, 'S3 Error')
+    def get(self):
+        client = getClientSafely()
+        if client is None:
+            abort(500, "S3 failed to start")
+        choice = random.choice(tuple(client.getCurrentMemeList()))
+
+        response = make_response(client.getMeme(choice))
+        response.headers['X-Meme-Name'] = choice
+
+        return response
+
+@api.route('/psuedorandom')
+@api.doc(description="Returns a psuedorandom meme. Will not return the same meme for a set number of requests")
+class getRandomFile(Resource):
+    cache = []
+    maxSize = 100
+    @api.doc('get')
+    @api.response(200, 'Sucess')
+    @api.response(500, 'S3 Error')
+    def get(self):
+        client = getClientSafely()
+        if client is None:
+            abort(500, "S3 failed to start")
+
+        choice = random.choice(tuple(client.getCurrentMemeList()))
+        while choice in self.cache:
+            choice = random.choice(tuple(client.getCurrentMemeList()))
+
+        self.cache.append(choice)
+
+        if len(self.cache) > self.maxSize:
+            self.cache.pop()
+
+        logging.debug("Contents of cache : " + str(self.cache))
+
+        response = make_response(client.getMeme(choice))
+        response.headers['X-Meme-Name'] = choice
+
+        return response
+
+@api.route('/share/<string:file_name>')
+@api.doc(description="Returns a share URL from the underlying bucket")
+class getShareLink(Resource):
+    @api.response(200, 'Sucess')
+    @api.response(500, 'S3 Error')
+    @api.response(404, 'Requested file not found')
+    def get(self, file_name):
+        client = getClientSafely()
+        if client is None:
+            abort(500, "S3 failed to start")
+
+        if file_name in client.getCurrentMemeList():
+            url = client.getShareForMeme(file_name)
+            return {
+                    "url": url,
+                    }
+        else:
+            abort(400, "Requested file '" + file_name + "' not found")
+
+
+

src/api/search.py

@@ -1,21 +1,14 @@
 from flask_restx import Namespace, Resource, fields
 from flask_restx import reqparse
-import s3Client
+from api.clientGetter import getClientSafely
+from api.get import getExactFile as getApi
+from flask import abort
+import nlp
 import logging
 
 # Exported namespace
 api = Namespace('search', description='Searching for memes')
 
-def getClientSafely():
-    logging.debug("Getting a client safely...")
-    client = None
-    try:
-        client = s3Client.getClient()
-    except Exception as e:
-        logging.critical("Failed to retrive a client : " + str(e))
-
-    return client
-
 
 @api.route('/exact/<string:query>')
 @api.doc(params={
@@ -26,14 +19,50 @@ class exactSearch(Resource):
     @api.response(200, 'Sucess')
     @api.response(500, 'S3 Error')
     def get(self, query):
-        logging.debug("Getting a client safely...")
         client = getClientSafely()
         if client is None:
-            return {
+            abort(500, "S3 failed to start")
-                    "message": "Error connecting to S3"
-                }, 500
         if query in client.getCurrentMemeList():
-            return "nice"
+            return {
+                    "found" : True,
+                    "url"   : "/resource/exact/" + query,
+                    "name"  : query,
+                    "tags"  : client.getTagsOnMeme(query),
+                }
         else:
-            return "boo"
+            return { "found" : False }
+
+@api.route('/close/<string:query>')
+@api.route('/<string:query>', doc={
+    "description" : "Alias for /close/{query}"
+    })
+@api.doc(params={
+    'query' : 'Search query to attempt to compare against'
+    },description="Find a meme thats close using levenshtein distance")
+class textualClose(Resource):
+    @api.doc('fuzzy search')
+    @api.response(200, 'Sucess')
+    @api.response(500, 'S3 Error')
+    def get(self, query):
+        client = getClientSafely()
+        if client is None:
+            abort(500, "S3 failed to start")
+
+        foundMemes = nlp.getCloseMemes(client.getCurrentMemeList(), query)
+        listToReturn = []
+
+        for meme in foundMemes:
+            entry = {
+                    "found" : True,
+                    "url"   : "/resource/exact/" + meme,
+                    "name"  : meme,
+                    "tags"  : client.getTagsOnMeme(meme),
+                }
+            listToReturn.append(entry)
+
+        return {
+                "results" : listToReturn,
+                "numberOfResults" : len(foundMemes)
+            }
+
 

src/api/util.py

@@ -0,0 +1,20 @@
+from flask_restx import Namespace, Resource, fields
+from flask_restx import reqparse
+from flask import abort
+from api.clientGetter import getClientSafely
+import logging
+
+# Exported namespace
+api = Namespace('util', description='Misc Utilities')
+
+@api.route('/count')
+@api.doc(description="Get number of memes in store")
+class getCount(Resource):
+    def get(self):
+        client = getClientSafely()
+        if client is None:
+            abort(500, "S3 failed to start")
+        return {
+                "count" : len(client.getCurrentMemeList())
+            }, 200
+

src/app.py

@@ -6,12 +6,13 @@ from api import api
 import os
 import logging
 
-logging.basicConfig(level=logging.INFO)
+logging.basicConfig(level=logging.DEBUG)
 
-isDebug = True
+isDebug = bool(os.environ.get('DEBUG', False))
+port = int(os.environ.get('PORT', 5000))
 
 app = Flask(__name__)
 
 api.init_app(app)
 
-app.run(debug=isDebug)
+app.run(debug=isDebug, host='0.0.0.0', port=port)

src/config.py

@@ -0,0 +1,7 @@
+# Load configuration file
+#
+
+import envyaml
+import os
+
+config = envyaml.EnvYAML(os.environ.get('CONFIG_PATH', 'config.yaml'))

src/config.yaml

@@ -0,0 +1,6 @@
+s3:
+  url: s3.clortox.com
+  username: ${S3_USERNAME}
+  password: ${S3_PASSWORD}
+  tls: True
+  bucket: memes

src/nlp.py

@@ -0,0 +1,32 @@
+from fuzzywuzzy import fuzz, process
+from api.clientGetter import getClientSafely
+import logging
+
+
+def getCloseMemes(allPossibleMemes: set, query: str):
+    if not isinstance(allPossibleMemes, set):
+        raise Exception("Expected set for allPossibleMemes")
+    if not isinstance(query, str):
+        raise Exception("Expected str for query")
+
+    topMeme = ''
+    topMemes = []
+    topScore = 0
+
+    for meme in allPossibleMemes:
+        currentScore = fuzz.partial_ratio(query, meme)
+        if currentScore > topScore:
+            topMeme = meme
+            topScore = currentScore
+        if currentScore == 100:
+            topMemes.append(meme)
+            topMeme = meme
+
+    logging.info('Top memes for given query (' + query + ")")
+    logging.info('topMemes: ' + str(topMemes))
+    logging.info('topMeme : ' + topMeme)
+
+    if len(topMemes) == 0:
+        topMemes.append(topMeme)
+
+    return topMemes

src/requirements.txt

@@ -1,12 +1,21 @@
 aniso8601==9.0.1
 attrs==22.2.0
+certifi==2022.12.7
 click==8.1.3
+envyaml==1.10.211231
 Flask==2.2.3
 flask-restx==1.0.6
+fuzzywuzzy==0.18.0
 itsdangerous==2.1.2
 Jinja2==3.1.2
 jsonschema==4.17.3
+Levenshtein==0.20.9
 MarkupSafe==2.1.2
+minio==7.1.13
 pyrsistent==0.19.3
+python-Levenshtein==0.20.9
 pytz==2022.7.1
+PyYAML==6.0
+rapidfuzz==2.13.7
+urllib3==1.26.14
 Werkzeug==2.2.3

src/s3Client.py

@@ -1,19 +1,22 @@
 import logging
 import os
 
-
+from minio.commonconfig import Tags
-
 from minio import Minio
 from minio.commonconfig import Tags
 from minio.error import S3Error
+from config import config
 from datetime import datetime
 
+from functools import lru_cache
+
 
 S3_URL = ""
 S3_UN  = ""
 S3_PW  = ""
 S3_TLS = True
 S3_BUCKET = ""
+CACHE_TTL = 10
 
 gclient = None
 
@@ -27,30 +30,30 @@ def getClient():
     if gclient != None:
         return gclient
 
-    if "S3_URL" not in os.environ:
+    if "url" not in config["s3"]:
         raise Exception("S3_URL is not set!")
-    S3_URL = os.environ["S3_URL"]
+    S3_URL = config["s3.url"]
     logging.info("Using S3_URL : " + S3_URL )
 
 
-    if "S3_UN" not in os.environ:
+    if "username" not in config["s3"]:
         raise Exception("S3_UN is not set!")
-    S3_UN = os.environ["S3_UN"]
+    S3_UN = config["s3.username"]
     logging.info("Using S3_UN : " + S3_UN)
 
-    if "S3_PW" not in os.environ:
+    if "password" not in config["s3"]:
         raise Exception("S3_PW is not set!")
-    S3_PW = os.environ["S3_PW"]
+    S3_PW = config["s3.password"]
     logging.info("Using S3_PW : " + S3_PW)
 
-    if "S3_BUCKET" not in os.environ:
+    if "bucket" not in config["s3"]:
         raise Exception("S3_BUCKET is not set!")
-    S3_BUCKET = os.environ["S3_BUCKET"]
+    S3_BUCKET = config["s3.bucket"]
     logging.info("Using S3_BUCKET : " + S3_BUCKET)
 
     # override defaults
-    if "S3_TLS" in os.environ:
+    if "tls" in config["s3"]:
-        S3_TLS = os.environ["S3_TLS"].lower() in ("yes", "true", "1", "t")
+        S3_TLS = config["s3.tls"]
     logging.info("Using S3_TLS : " + str(S3_TLS))
 
     client = Minio(S3_URL,
@@ -72,6 +75,7 @@ def getClient():
 class Client:
     allMemes = set()
     memesToMd5 = dict()
+    memesToTags = dict()
     lastCheckedAllMemes = datetime.strptime("2000-01-01 01:01:01", "%Y-%m-%d %H:%M:%S")
     client = None
 
@@ -84,8 +88,12 @@ class Client:
             self.client = client
         else:
             raise Exception("Improper object passed for client!")
+        self.getCurrentMemeList(force=True)
 
     def getCurrentMemeList(self, force=False):
+        """
+        Get a list of all memes in the bucket
+        """
         now = datetime.now()
         if (now - self.lastCheckedAllMemes).seconds > 300 or force:
             logging.info("Enough time has elapsed, refreshing meme cache...")
@@ -93,6 +101,7 @@ class Client:
 
             self.allMemes.clear()
             self.memesToMd5.clear()
+            self.memesToTags.clear()
 
             for obj in self.client.list_objects(S3_BUCKET):
                 if not obj.is_dir:
@@ -102,4 +111,84 @@ class Client:
 
         return self.allMemes
 
+    @lru_cache(maxsize=32)
+    def getMeme(self, memeName: str):
+        """
+        Return a meme with the exact given name, or raise an exception
+        """
+
+        if not isinstance(memeName, str):
+            raise Exception("paramater memeName is of improper type, expected a str")
+
+        memeSet = self.getCurrentMemeList()
+        if memeName in memeSet:
+            reply = self.client.get_object(bucket_name=S3_BUCKET,
+                                            object_name=memeName)
+            return reply.read()
+        else:
+            raise Exception("Requested meme '" + memeName + "' not found")
+        return None
+
+    def addMeme(self, fileContents, name: str, tags: Tags = Tags.new_object_tags()):
+        result = self.client.put_object(bucket_name=S3_BUCKET,
+                                        object_name=name,
+                                        data=fileContents,
+                                        length=-1,
+                                        tags=tags,
+                                        part_size=10*1024*1024)
+        if result.etag in self.memesToMd5:
+            logging.info('Uploaded meme named ' + name + ' already exists')
+            self.client.remove_object(bucket_name=S3_BUCKET,
+                                 object_name=name)
+            return False
+        else:
+            self.allMemes.add(name)
+            self.memesToMd5[name] = result.etag
+            self.memesToTags[name] = tags
+            return True
+
+    @lru_cache(maxsize=32)
+    def getTagsOnMeme(self, memeName: str):
+        """
+        Returns the S3 Tags object for a given meme
+        """
+
+        if not isinstance(memeName, str):
+            raise Exception("paramater memeName is of improper type, expected a str")
+
+        if memeName in self.memesToTags:
+            return self.memesToTags[memeName]
+
+
+        memeSet = self.getCurrentMemeList()
+        if memeName in memeSet:
+            reply = self.client.get_object_tags(bucket_name=S3_BUCKET,
+                                            object_name=memeName)
+            self.memesToTags[memeName] = reply
+            return reply
+        else:
+            raise Exception("Requested meme '" + memeName + "' not found")
+        return None
+
+    def getShareForMeme(self, memeName: str) -> str:
+        """
+        Returns the S3 bucket's share link for the meme
+        """
+
+        if not isinstance(memeName, str):
+            raise Exception("paramater memeName is of improper type, expected a str")
+
+        if memeName not in self.getCurrentMemeList():
+            raise Exception("Requested meme '" + memeName + "' not found")
+
+        reply = self.client.get_presigned_url(
+                method="GET",
+                bucket_name=S3_BUCKET,
+                object_name=memeName)
+
+        return reply
+
+
+
+