0x0

mia's file "the null pointer" hosting application -- l0bster h0sted
Log | Files | Refs | LICENSE

fhost.py (15478B)


      1 #!/usr/bin/env python3
      2 # -*- coding: utf-8 -*-
      3 
      4 from flask import Flask, abort, escape, make_response, redirect, request, send_from_directory, url_for, Response
      5 from flask_sqlalchemy import SQLAlchemy
      6 from flask_script import Manager
      7 from flask_migrate import Migrate, MigrateCommand
      8 from hashlib import sha256
      9 from humanize import naturalsize
     10 from magic import Magic
     11 from mimetypes import guess_extension
     12 import os, sys
     13 import requests
     14 from short_url import UrlEncoder
     15 from validators import url as url_valid
     16 
     17 app = Flask(__name__)
     18 app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
     19 
     20 app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///db.sqlite" # "postgresql://0x0@/0x0"
     21 app.config["PREFERRED_URL_SCHEME"] = "https" # nginx users: make sure to have 'uwsgi_param UWSGI_SCHEME $scheme;' in your config
     22 app.config["MAX_CONTENT_LENGTH"] = 256 * 1024 * 1024
     23 app.config["MAX_URL_LENGTH"] = 4096
     24 app.config["FHOST_STORAGE_PATH"] = "up"
     25 app.config["FHOST_USE_X_ACCEL_REDIRECT"] = True # expect nginx by default
     26 app.config["USE_X_SENDFILE"] = False
     27 app.config["FHOST_EXT_OVERRIDE"] = {
     28     "audio/flac" : ".flac",
     29     "image/gif" : ".gif",
     30     "image/jpeg" : ".jpg",
     31     "image/png" : ".png",
     32     "image/svg+xml" : ".svg",
     33     "video/webm" : ".webm",
     34     "video/x-matroska" : ".mkv",
     35     "application/octet-stream" : ".bin",
     36     "text/plain" : ".txt",
     37     "text/x-diff" : ".diff",
     38 }
     39 
     40 # default blacklist to avoid AV mafia extortion
     41 app.config["FHOST_MIME_BLACKLIST"] = [
     42     "application/x-dosexec",
     43     "application/java-archive",
     44     "application/java-vm"
     45 ]
     46 
     47 app.config["FHOST_UPLOAD_BLACKLIST"] = "tornodes.txt"
     48 
     49 app.config["NSFW_DETECT"] = False
     50 app.config["NSFW_THRESHOLD"] = 0.608
     51 
     52 if app.config["NSFW_DETECT"]:
     53     from nsfw_detect import NSFWDetector
     54     nsfw = NSFWDetector()
     55 
     56 try:
     57     mimedetect = Magic(mime=True, mime_encoding=False)
     58 except:
     59     print("""Error: You have installed the wrong version of the 'magic' module.
     60 Please install python-magic.""")
     61     sys.exit(1)
     62 
     63 if not os.path.exists(app.config["FHOST_STORAGE_PATH"]):
     64     os.mkdir(app.config["FHOST_STORAGE_PATH"])
     65 
     66 db = SQLAlchemy(app)
     67 migrate = Migrate(app, db)
     68 
     69 manager = Manager(app)
     70 manager.add_command("db", MigrateCommand)
     71 
     72 su = UrlEncoder(alphabet='DEQhd2uFteibPwq0SWBInTpA_jcZL5GKz3YCR14Ulk87Jors9vNHgfaOmMXy6Vx-', block_size=16)
     73 
     74 class URL(db.Model):
     75     id = db.Column(db.Integer, primary_key = True)
     76     url = db.Column(db.UnicodeText, unique = True)
     77 
     78     def __init__(self, url):
     79         self.url = url
     80 
     81     def getname(self):
     82         return su.enbase(self.id, 1)
     83 
     84     def geturl(self):
     85         return url_for("get", path=self.getname(), _external=True) + "\n"
     86 
     87 class File(db.Model):
     88     id = db.Column(db.Integer, primary_key = True)
     89     sha256 = db.Column(db.String, unique = True)
     90     ext = db.Column(db.UnicodeText)
     91     mime = db.Column(db.UnicodeText)
     92     addr = db.Column(db.UnicodeText)
     93     removed = db.Column(db.Boolean, default=False)
     94     nsfw_score = db.Column(db.Float)
     95 
     96     def __init__(self, sha256, ext, mime, addr, nsfw_score):
     97         self.sha256 = sha256
     98         self.ext = ext
     99         self.mime = mime
    100         self.addr = addr
    101         self.nsfw_score = nsfw_score
    102 
    103     def getname(self):
    104         return u"{0}{1}".format(su.enbase(self.id, 1), self.ext)
    105 
    106     def geturl(self):
    107         n = self.getname()
    108 
    109         if self.nsfw_score and self.nsfw_score > app.config["NSFW_THRESHOLD"]:
    110             return url_for("get", path=n, _external=True, _anchor="nsfw") + "\n"
    111         else:
    112             return url_for("get", path=n, _external=True) + "\n"
    113 
    114     def pprint(self):
    115         print("url: {}".format(self.getname()))
    116         vals = vars(self)
    117 
    118         for v in vals:
    119             if not v.startswith("_sa"):
    120                 print("{}: {}".format(v, vals[v]))
    121 
    122 def getpath(fn):
    123     return os.path.join(app.config["FHOST_STORAGE_PATH"], fn)
    124 
    125 def fhost_url(scheme=None):
    126     if not scheme:
    127         return url_for(".fhost", _external=True).rstrip("/")
    128     else:
    129         return url_for(".fhost", _external=True, _scheme=scheme).rstrip("/")
    130 
    131 def is_fhost_url(url):
    132     return url.startswith(fhost_url()) or url.startswith(fhost_url("https"))
    133 
    134 def shorten(url):
    135     if len(url) > app.config["MAX_URL_LENGTH"]:
    136         abort(414)
    137 
    138     if not url_valid(url) or is_fhost_url(url) or "\n" in url:
    139         abort(400)
    140 
    141     existing = URL.query.filter_by(url=url).first()
    142 
    143     if existing:
    144         return existing.geturl()
    145     else:
    146         u = URL(url)
    147         db.session.add(u)
    148         db.session.commit()
    149 
    150         return u.geturl()
    151 
    152 def in_upload_bl(addr):
    153     if os.path.isfile(app.config["FHOST_UPLOAD_BLACKLIST"]):
    154         with open(app.config["FHOST_UPLOAD_BLACKLIST"], "r") as bl:
    155             check = addr.lstrip("::ffff:")
    156             for l in bl.readlines():
    157                 if not l.startswith("#"):
    158                     if check == l.rstrip():
    159                         return True
    160 
    161     return False
    162 
    163 def store_file(f, addr):
    164     if in_upload_bl(addr):
    165         return "Your host is blocked from uploading files.\n", 451
    166 
    167     data = f.stream.read()
    168     digest = sha256(data).hexdigest()
    169     existing = File.query.filter_by(sha256=digest).first()
    170 
    171     if existing:
    172         if existing.removed:
    173             return legal()
    174 
    175         epath = getpath(existing.sha256)
    176 
    177         if not os.path.exists(epath):
    178             with open(epath, "wb") as of:
    179                 of.write(data)
    180 
    181         if existing.nsfw_score == None:
    182             if app.config["NSFW_DETECT"]:
    183                 existing.nsfw_score = nsfw.detect(epath)
    184 
    185         os.utime(epath, None)
    186         existing.addr = addr
    187         db.session.commit()
    188 
    189         return existing.geturl()
    190     else:
    191         guessmime = mimedetect.from_buffer(data)
    192 
    193         if not f.content_type or not "/" in f.content_type or f.content_type == "application/octet-stream":
    194             mime = guessmime
    195         else:
    196             mime = f.content_type
    197 
    198         if mime in app.config["FHOST_MIME_BLACKLIST"] or guessmime in app.config["FHOST_MIME_BLACKLIST"]:
    199             abort(415)
    200 
    201         if mime.startswith("text/") and not "charset" in mime:
    202             mime += "; charset=utf-8"
    203 
    204         ext = os.path.splitext(f.filename)[1]
    205 
    206         if not ext:
    207             gmime = mime.split(";")[0]
    208 
    209             if not gmime in app.config["FHOST_EXT_OVERRIDE"]:
    210                 ext = guess_extension(gmime)
    211             else:
    212                 ext = app.config["FHOST_EXT_OVERRIDE"][gmime]
    213         else:
    214             ext = ext[:8]
    215 
    216         if not ext:
    217             ext = ".bin"
    218 
    219         spath = getpath(digest)
    220 
    221         with open(spath, "wb") as of:
    222             of.write(data)
    223 
    224         if app.config["NSFW_DETECT"]:
    225             nsfw_score = nsfw.detect(spath)
    226         else:
    227             nsfw_score = None
    228 
    229         sf = File(digest, ext, mime, addr, nsfw_score)
    230         db.session.add(sf)
    231         db.session.commit()
    232 
    233         return sf.geturl()
    234 
    235 def store_url(url, addr):
    236     if is_fhost_url(url):
    237         return segfault(508)
    238 
    239     h = { "Accept-Encoding" : "identity" }
    240     r = requests.get(url, stream=True, verify=False, headers=h)
    241 
    242     try:
    243         r.raise_for_status()
    244     except requests.exceptions.HTTPError as e:
    245         return str(e) + "\n"
    246 
    247     if "content-length" in r.headers:
    248         l = int(r.headers["content-length"])
    249 
    250         if l < app.config["MAX_CONTENT_LENGTH"]:
    251             def urlfile(**kwargs):
    252                 return type('',(),kwargs)()
    253 
    254             f = urlfile(stream=r.raw, content_type=r.headers["content-type"], filename="")
    255 
    256             return store_file(f, addr)
    257         else:
    258             hl = naturalsize(l, binary = True)
    259             hml = naturalsize(app.config["MAX_CONTENT_LENGTH"], binary=True)
    260 
    261             return "Remote file too large ({0} > {1}).\n".format(hl, hml), 413
    262     else:
    263         return "Could not determine remote file size (no Content-Length in response header; shoot admin).\n", 411
    264 
    265 @app.route("/<path:path>")
    266 def get(path):
    267     p = os.path.splitext(path)
    268     id = su.debase(p[0])
    269 
    270     if p[1]:
    271         f = File.query.get(id)
    272 
    273         if f and f.ext == p[1]:
    274             if f.removed:
    275                 return legal()
    276 
    277             fpath = getpath(f.sha256)
    278 
    279             if not os.path.exists(fpath):
    280                 abort(404)
    281 
    282             fsize = os.path.getsize(fpath)
    283 
    284             if app.config["FHOST_USE_X_ACCEL_REDIRECT"]:
    285                 response = make_response()
    286                 response.headers["Content-Type"] = f.mime
    287                 response.headers["Content-Length"] = fsize
    288                 response.headers["X-Accel-Redirect"] = "/" + fpath
    289                 return response
    290             else:
    291                 return send_from_directory(app.config["FHOST_STORAGE_PATH"], f.sha256, mimetype = f.mime)
    292     else:
    293         u = URL.query.get(id)
    294 
    295         if u:
    296             return redirect(u.url)
    297 
    298     abort(404)
    299 
    300 @app.route("/dump_urls/")
    301 @app.route("/dump_urls/<int:start>")
    302 def dump_urls(start=0):
    303     meta = "#FORMAT: BEACON\n#PREFIX: {}/\n\n".format(fhost_url("https"))
    304 
    305     def gen():
    306         yield meta
    307 
    308         for url in URL.query.order_by(URL.id.asc()).offset(start):
    309             if url.url.startswith("http") or url.url.startswith("https"):
    310                 bar = "|"
    311             else:
    312                 bar = "||"
    313 
    314             yield url.getname() + bar + url.url + "\n"
    315 
    316     return Response(gen(), mimetype="text/plain")
    317 
    318 @app.route("/", methods=["GET", "POST"])
    319 def fhost():
    320     if request.method == "POST":
    321         sf = None
    322 
    323         if "file" in request.files:
    324             return store_file(request.files["file"], request.remote_addr)
    325         elif "url" in request.form:
    326             return store_url(request.form["url"], request.remote_addr)
    327         elif "shorten" in request.form:
    328             return shorten(request.form["shorten"])
    329 
    330         abort(400)
    331     else:
    332         fmts = list(app.config["FHOST_EXT_OVERRIDE"])
    333         fmts.sort()
    334         maxsize = naturalsize(app.config["MAX_CONTENT_LENGTH"], binary=True)
    335         maxsizenum, maxsizeunit = maxsize.split(" ")
    336         maxsizenum = float(maxsizenum)
    337         maxsizehalf = maxsizenum / 2
    338 
    339         if maxsizenum.is_integer():
    340             maxsizenum = int(maxsizenum)
    341         if maxsizehalf.is_integer():
    342             maxsizehalf = int(maxsizehalf)
    343 
    344         return """<pre>
    345 THE NULL POINTER
    346 ================
    347 
    348 HTTP POST files here:
    349     curl -F'file=@yourfile.png' {0}
    350 You can also POST remote URLs:
    351     curl -F'url=http://example.com/image.jpg' {0}
    352 Or you can shorten URLs:
    353     curl -F'shorten=http://example.com/some/long/url' {0}
    354 
    355 File URLs are valid for at least 30 days and up to a year (see below).
    356 Shortened URLs do not expire.
    357 
    358 Maximum file size: {1}
    359 Not allowed: {5}
    360 
    361 
    362 FILE RETENTION PERIOD
    363 ---------------------
    364 
    365 retention = min_age + (-max_age + min_age) * pow((file_size / max_size - 1), 3)
    366 
    367    days
    368     365 |  \\
    369         |   \\
    370         |    \\
    371         |     \\
    372         |      \\
    373         |       \\
    374         |        ..
    375         |          \\
    376   197.5 | ----------..-------------------------------------------
    377         |             ..
    378         |               \\
    379         |                ..
    380         |                  ...
    381         |                     ..
    382         |                       ...
    383         |                          ....
    384         |                              ......
    385      30 |                                    ....................
    386           0{2}{3}
    387            {4}
    388 
    389 
    390 ABUSE
    391 -----
    392 
    393 If you would like to request permanent deletion, please contact l0bster via
    394 irc.l0bster.ru, or send an email to 0x0@underd0g.co.
    395 
    396 Please allow up to 24 hours for a response.
    397 </pre>
    398 """.format(fhost_url(),
    399            maxsize, str(maxsizehalf).rjust(27), str(maxsizenum).rjust(27),
    400            maxsizeunit.rjust(54),
    401            ", ".join(app.config["FHOST_MIME_BLACKLIST"]))
    402 
    403 @app.route("/robots.txt")
    404 def robots():
    405     return """User-agent: *
    406 Disallow: /
    407 """
    408 
    409 def legal():
    410     return "451 Unavailable For Legal Reasons\n", 451
    411 
    412 @app.errorhandler(400)
    413 @app.errorhandler(404)
    414 @app.errorhandler(414)
    415 @app.errorhandler(415)
    416 def segfault(e):
    417     return "Segmentation fault\n", e.code
    418 
    419 @app.errorhandler(404)
    420 def notfound(e):
    421     return u"""<pre>Process {0} stopped
    422 * thread #1: tid = {0}, {1:#018x}, name = '{2}'
    423     frame #0:
    424 Process {0} stopped
    425 * thread #8: tid = {0}, {3:#018x} fhost`get(path='{4}') + 27 at fhost.c:139, name = 'fhost/responder', stop reason = invalid address (fault address: 0x30)
    426     frame #0: {3:#018x} fhost`get(path='{4}') + 27 at fhost.c:139
    427    136   get(SrvContext *ctx, const char *path)
    428    137   {{
    429    138       StoredObj *obj = ctx->store->query(shurl_debase(path));
    430 -> 139       switch (obj->type) {{
    431    140           case ObjTypeFile:
    432    141               ctx->serve_file_id(obj->id);
    433    142               break;
    434 (lldb) q</pre>
    435 """.format(os.getpid(), id(app), "fhost", id(get), escape(request.path)), e.code
    436 
    437 @manager.command
    438 def debug():
    439     app.config["FHOST_USE_X_ACCEL_REDIRECT"] = False
    440     app.run(debug=True, port=4562,host="0.0.0.0")
    441 
    442 @manager.command
    443 def permadelete(name):
    444     id = su.debase(name)
    445     f = File.query.get(id)
    446 
    447     if f:
    448         if os.path.exists(getpath(f.sha256)):
    449             os.remove(getpath(f.sha256))
    450         f.removed = True
    451         db.session.commit()
    452 
    453 @manager.command
    454 def query(name):
    455     id = su.debase(name)
    456     f = File.query.get(id)
    457 
    458     if f:
    459         f.pprint()
    460 
    461 @manager.command
    462 def queryhash(h):
    463     f = File.query.filter_by(sha256=h).first()
    464 
    465     if f:
    466         f.pprint()
    467 
    468 @manager.command
    469 def queryaddr(a, nsfw=False, removed=False):
    470     res = File.query.filter_by(addr=a)
    471 
    472     if not removed:
    473         res = res.filter(File.removed != True)
    474 
    475     if nsfw:
    476         res = res.filter(File.nsfw_score > app.config["NSFW_THRESHOLD"])
    477 
    478     for f in res:
    479         f.pprint()
    480 
    481 @manager.command
    482 def deladdr(a):
    483     res = File.query.filter_by(addr=a).filter(File.removed != True)
    484 
    485     for f in res:
    486         if os.path.exists(getpath(f.sha256)):
    487             os.remove(getpath(f.sha256))
    488         f.removed = True
    489 
    490     db.session.commit()
    491 
    492 def nsfw_detect(f):
    493     try:
    494         open(f["path"], 'r').close()
    495         f["nsfw_score"] = nsfw.detect(f["path"])
    496         return f
    497     except:
    498         return None
    499 
    500 @manager.command
    501 def update_nsfw():
    502     if not app.config["NSFW_DETECT"]:
    503         print("NSFW detection is disabled in app config")
    504         return 1
    505 
    506     from multiprocessing import Pool
    507     import tqdm
    508 
    509     res = File.query.filter_by(nsfw_score=None, removed=False)
    510 
    511     with Pool() as p:
    512         results = []
    513         work = [{ "path" : getpath(f.sha256), "id" : f.id} for f in res]
    514 
    515         for r in tqdm.tqdm(p.imap_unordered(nsfw_detect, work), total=len(work)):
    516             if r:
    517                 results.append({"id": r["id"], "nsfw_score" : r["nsfw_score"]})
    518 
    519         db.session.bulk_update_mappings(File, results)
    520         db.session.commit()
    521 
    522 
    523 @manager.command
    524 def querybl(nsfw=False, removed=False):
    525     blist = []
    526     if os.path.isfile(app.config["FHOST_UPLOAD_BLACKLIST"]):
    527         with open(app.config["FHOST_UPLOAD_BLACKLIST"], "r") as bl:
    528             for l in bl.readlines():
    529                 if not l.startswith("#"):
    530                     if not ":" in l:
    531                         blist.append("::ffff:" + l.rstrip())
    532                     else:
    533                         blist.append(l.strip())
    534 
    535     res = File.query.filter(File.addr.in_(blist))
    536 
    537     if not removed:
    538         res = res.filter(File.removed != True)
    539 
    540     if nsfw:
    541         res = res.filter(File.nsfw_score > app.config["NSFW_THRESHOLD"])
    542 
    543     for f in res:
    544         f.pprint()
    545 
    546 if __name__ == "__main__":
    547     manager.run()