fhost.py (15478B)
1 #!/usr/bin/env python3 2 # -*- coding: utf-8 -*- 3 4 from flask import Flask, abort, escape, make_response, redirect, request, send_from_directory, url_for, Response 5 from flask_sqlalchemy import SQLAlchemy 6 from flask_script import Manager 7 from flask_migrate import Migrate, MigrateCommand 8 from hashlib import sha256 9 from humanize import naturalsize 10 from magic import Magic 11 from mimetypes import guess_extension 12 import os, sys 13 import requests 14 from short_url import UrlEncoder 15 from validators import url as url_valid 16 17 app = Flask(__name__) 18 app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False 19 20 app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///db.sqlite" # "postgresql://0x0@/0x0" 21 app.config["PREFERRED_URL_SCHEME"] = "https" # nginx users: make sure to have 'uwsgi_param UWSGI_SCHEME $scheme;' in your config 22 app.config["MAX_CONTENT_LENGTH"] = 256 * 1024 * 1024 23 app.config["MAX_URL_LENGTH"] = 4096 24 app.config["FHOST_STORAGE_PATH"] = "up" 25 app.config["FHOST_USE_X_ACCEL_REDIRECT"] = True # expect nginx by default 26 app.config["USE_X_SENDFILE"] = False 27 app.config["FHOST_EXT_OVERRIDE"] = { 28 "audio/flac" : ".flac", 29 "image/gif" : ".gif", 30 "image/jpeg" : ".jpg", 31 "image/png" : ".png", 32 "image/svg+xml" : ".svg", 33 "video/webm" : ".webm", 34 "video/x-matroska" : ".mkv", 35 "application/octet-stream" : ".bin", 36 "text/plain" : ".txt", 37 "text/x-diff" : ".diff", 38 } 39 40 # default blacklist to avoid AV mafia extortion 41 app.config["FHOST_MIME_BLACKLIST"] = [ 42 "application/x-dosexec", 43 "application/java-archive", 44 "application/java-vm" 45 ] 46 47 app.config["FHOST_UPLOAD_BLACKLIST"] = "tornodes.txt" 48 49 app.config["NSFW_DETECT"] = False 50 app.config["NSFW_THRESHOLD"] = 0.608 51 52 if app.config["NSFW_DETECT"]: 53 from nsfw_detect import NSFWDetector 54 nsfw = NSFWDetector() 55 56 try: 57 mimedetect = Magic(mime=True, mime_encoding=False) 58 except: 59 print("""Error: You have installed the wrong version of the 'magic' module. 60 Please install python-magic.""") 61 sys.exit(1) 62 63 if not os.path.exists(app.config["FHOST_STORAGE_PATH"]): 64 os.mkdir(app.config["FHOST_STORAGE_PATH"]) 65 66 db = SQLAlchemy(app) 67 migrate = Migrate(app, db) 68 69 manager = Manager(app) 70 manager.add_command("db", MigrateCommand) 71 72 su = UrlEncoder(alphabet='DEQhd2uFteibPwq0SWBInTpA_jcZL5GKz3YCR14Ulk87Jors9vNHgfaOmMXy6Vx-', block_size=16) 73 74 class URL(db.Model): 75 id = db.Column(db.Integer, primary_key = True) 76 url = db.Column(db.UnicodeText, unique = True) 77 78 def __init__(self, url): 79 self.url = url 80 81 def getname(self): 82 return su.enbase(self.id, 1) 83 84 def geturl(self): 85 return url_for("get", path=self.getname(), _external=True) + "\n" 86 87 class File(db.Model): 88 id = db.Column(db.Integer, primary_key = True) 89 sha256 = db.Column(db.String, unique = True) 90 ext = db.Column(db.UnicodeText) 91 mime = db.Column(db.UnicodeText) 92 addr = db.Column(db.UnicodeText) 93 removed = db.Column(db.Boolean, default=False) 94 nsfw_score = db.Column(db.Float) 95 96 def __init__(self, sha256, ext, mime, addr, nsfw_score): 97 self.sha256 = sha256 98 self.ext = ext 99 self.mime = mime 100 self.addr = addr 101 self.nsfw_score = nsfw_score 102 103 def getname(self): 104 return u"{0}{1}".format(su.enbase(self.id, 1), self.ext) 105 106 def geturl(self): 107 n = self.getname() 108 109 if self.nsfw_score and self.nsfw_score > app.config["NSFW_THRESHOLD"]: 110 return url_for("get", path=n, _external=True, _anchor="nsfw") + "\n" 111 else: 112 return url_for("get", path=n, _external=True) + "\n" 113 114 def pprint(self): 115 print("url: {}".format(self.getname())) 116 vals = vars(self) 117 118 for v in vals: 119 if not v.startswith("_sa"): 120 print("{}: {}".format(v, vals[v])) 121 122 def getpath(fn): 123 return os.path.join(app.config["FHOST_STORAGE_PATH"], fn) 124 125 def fhost_url(scheme=None): 126 if not scheme: 127 return url_for(".fhost", _external=True).rstrip("/") 128 else: 129 return url_for(".fhost", _external=True, _scheme=scheme).rstrip("/") 130 131 def is_fhost_url(url): 132 return url.startswith(fhost_url()) or url.startswith(fhost_url("https")) 133 134 def shorten(url): 135 if len(url) > app.config["MAX_URL_LENGTH"]: 136 abort(414) 137 138 if not url_valid(url) or is_fhost_url(url) or "\n" in url: 139 abort(400) 140 141 existing = URL.query.filter_by(url=url).first() 142 143 if existing: 144 return existing.geturl() 145 else: 146 u = URL(url) 147 db.session.add(u) 148 db.session.commit() 149 150 return u.geturl() 151 152 def in_upload_bl(addr): 153 if os.path.isfile(app.config["FHOST_UPLOAD_BLACKLIST"]): 154 with open(app.config["FHOST_UPLOAD_BLACKLIST"], "r") as bl: 155 check = addr.lstrip("::ffff:") 156 for l in bl.readlines(): 157 if not l.startswith("#"): 158 if check == l.rstrip(): 159 return True 160 161 return False 162 163 def store_file(f, addr): 164 if in_upload_bl(addr): 165 return "Your host is blocked from uploading files.\n", 451 166 167 data = f.stream.read() 168 digest = sha256(data).hexdigest() 169 existing = File.query.filter_by(sha256=digest).first() 170 171 if existing: 172 if existing.removed: 173 return legal() 174 175 epath = getpath(existing.sha256) 176 177 if not os.path.exists(epath): 178 with open(epath, "wb") as of: 179 of.write(data) 180 181 if existing.nsfw_score == None: 182 if app.config["NSFW_DETECT"]: 183 existing.nsfw_score = nsfw.detect(epath) 184 185 os.utime(epath, None) 186 existing.addr = addr 187 db.session.commit() 188 189 return existing.geturl() 190 else: 191 guessmime = mimedetect.from_buffer(data) 192 193 if not f.content_type or not "/" in f.content_type or f.content_type == "application/octet-stream": 194 mime = guessmime 195 else: 196 mime = f.content_type 197 198 if mime in app.config["FHOST_MIME_BLACKLIST"] or guessmime in app.config["FHOST_MIME_BLACKLIST"]: 199 abort(415) 200 201 if mime.startswith("text/") and not "charset" in mime: 202 mime += "; charset=utf-8" 203 204 ext = os.path.splitext(f.filename)[1] 205 206 if not ext: 207 gmime = mime.split(";")[0] 208 209 if not gmime in app.config["FHOST_EXT_OVERRIDE"]: 210 ext = guess_extension(gmime) 211 else: 212 ext = app.config["FHOST_EXT_OVERRIDE"][gmime] 213 else: 214 ext = ext[:8] 215 216 if not ext: 217 ext = ".bin" 218 219 spath = getpath(digest) 220 221 with open(spath, "wb") as of: 222 of.write(data) 223 224 if app.config["NSFW_DETECT"]: 225 nsfw_score = nsfw.detect(spath) 226 else: 227 nsfw_score = None 228 229 sf = File(digest, ext, mime, addr, nsfw_score) 230 db.session.add(sf) 231 db.session.commit() 232 233 return sf.geturl() 234 235 def store_url(url, addr): 236 if is_fhost_url(url): 237 return segfault(508) 238 239 h = { "Accept-Encoding" : "identity" } 240 r = requests.get(url, stream=True, verify=False, headers=h) 241 242 try: 243 r.raise_for_status() 244 except requests.exceptions.HTTPError as e: 245 return str(e) + "\n" 246 247 if "content-length" in r.headers: 248 l = int(r.headers["content-length"]) 249 250 if l < app.config["MAX_CONTENT_LENGTH"]: 251 def urlfile(**kwargs): 252 return type('',(),kwargs)() 253 254 f = urlfile(stream=r.raw, content_type=r.headers["content-type"], filename="") 255 256 return store_file(f, addr) 257 else: 258 hl = naturalsize(l, binary = True) 259 hml = naturalsize(app.config["MAX_CONTENT_LENGTH"], binary=True) 260 261 return "Remote file too large ({0} > {1}).\n".format(hl, hml), 413 262 else: 263 return "Could not determine remote file size (no Content-Length in response header; shoot admin).\n", 411 264 265 @app.route("/<path:path>") 266 def get(path): 267 p = os.path.splitext(path) 268 id = su.debase(p[0]) 269 270 if p[1]: 271 f = File.query.get(id) 272 273 if f and f.ext == p[1]: 274 if f.removed: 275 return legal() 276 277 fpath = getpath(f.sha256) 278 279 if not os.path.exists(fpath): 280 abort(404) 281 282 fsize = os.path.getsize(fpath) 283 284 if app.config["FHOST_USE_X_ACCEL_REDIRECT"]: 285 response = make_response() 286 response.headers["Content-Type"] = f.mime 287 response.headers["Content-Length"] = fsize 288 response.headers["X-Accel-Redirect"] = "/" + fpath 289 return response 290 else: 291 return send_from_directory(app.config["FHOST_STORAGE_PATH"], f.sha256, mimetype = f.mime) 292 else: 293 u = URL.query.get(id) 294 295 if u: 296 return redirect(u.url) 297 298 abort(404) 299 300 @app.route("/dump_urls/") 301 @app.route("/dump_urls/<int:start>") 302 def dump_urls(start=0): 303 meta = "#FORMAT: BEACON\n#PREFIX: {}/\n\n".format(fhost_url("https")) 304 305 def gen(): 306 yield meta 307 308 for url in URL.query.order_by(URL.id.asc()).offset(start): 309 if url.url.startswith("http") or url.url.startswith("https"): 310 bar = "|" 311 else: 312 bar = "||" 313 314 yield url.getname() + bar + url.url + "\n" 315 316 return Response(gen(), mimetype="text/plain") 317 318 @app.route("/", methods=["GET", "POST"]) 319 def fhost(): 320 if request.method == "POST": 321 sf = None 322 323 if "file" in request.files: 324 return store_file(request.files["file"], request.remote_addr) 325 elif "url" in request.form: 326 return store_url(request.form["url"], request.remote_addr) 327 elif "shorten" in request.form: 328 return shorten(request.form["shorten"]) 329 330 abort(400) 331 else: 332 fmts = list(app.config["FHOST_EXT_OVERRIDE"]) 333 fmts.sort() 334 maxsize = naturalsize(app.config["MAX_CONTENT_LENGTH"], binary=True) 335 maxsizenum, maxsizeunit = maxsize.split(" ") 336 maxsizenum = float(maxsizenum) 337 maxsizehalf = maxsizenum / 2 338 339 if maxsizenum.is_integer(): 340 maxsizenum = int(maxsizenum) 341 if maxsizehalf.is_integer(): 342 maxsizehalf = int(maxsizehalf) 343 344 return """<pre> 345 THE NULL POINTER 346 ================ 347 348 HTTP POST files here: 349 curl -F'file=@yourfile.png' {0} 350 You can also POST remote URLs: 351 curl -F'url=http://example.com/image.jpg' {0} 352 Or you can shorten URLs: 353 curl -F'shorten=http://example.com/some/long/url' {0} 354 355 File URLs are valid for at least 30 days and up to a year (see below). 356 Shortened URLs do not expire. 357 358 Maximum file size: {1} 359 Not allowed: {5} 360 361 362 FILE RETENTION PERIOD 363 --------------------- 364 365 retention = min_age + (-max_age + min_age) * pow((file_size / max_size - 1), 3) 366 367 days 368 365 | \\ 369 | \\ 370 | \\ 371 | \\ 372 | \\ 373 | \\ 374 | .. 375 | \\ 376 197.5 | ----------..------------------------------------------- 377 | .. 378 | \\ 379 | .. 380 | ... 381 | .. 382 | ... 383 | .... 384 | ...... 385 30 | .................... 386 0{2}{3} 387 {4} 388 389 390 ABUSE 391 ----- 392 393 If you would like to request permanent deletion, please contact l0bster via 394 irc.l0bster.ru, or send an email to 0x0@underd0g.co. 395 396 Please allow up to 24 hours for a response. 397 </pre> 398 """.format(fhost_url(), 399 maxsize, str(maxsizehalf).rjust(27), str(maxsizenum).rjust(27), 400 maxsizeunit.rjust(54), 401 ", ".join(app.config["FHOST_MIME_BLACKLIST"])) 402 403 @app.route("/robots.txt") 404 def robots(): 405 return """User-agent: * 406 Disallow: / 407 """ 408 409 def legal(): 410 return "451 Unavailable For Legal Reasons\n", 451 411 412 @app.errorhandler(400) 413 @app.errorhandler(404) 414 @app.errorhandler(414) 415 @app.errorhandler(415) 416 def segfault(e): 417 return "Segmentation fault\n", e.code 418 419 @app.errorhandler(404) 420 def notfound(e): 421 return u"""<pre>Process {0} stopped 422 * thread #1: tid = {0}, {1:#018x}, name = '{2}' 423 frame #0: 424 Process {0} stopped 425 * thread #8: tid = {0}, {3:#018x} fhost`get(path='{4}') + 27 at fhost.c:139, name = 'fhost/responder', stop reason = invalid address (fault address: 0x30) 426 frame #0: {3:#018x} fhost`get(path='{4}') + 27 at fhost.c:139 427 136 get(SrvContext *ctx, const char *path) 428 137 {{ 429 138 StoredObj *obj = ctx->store->query(shurl_debase(path)); 430 -> 139 switch (obj->type) {{ 431 140 case ObjTypeFile: 432 141 ctx->serve_file_id(obj->id); 433 142 break; 434 (lldb) q</pre> 435 """.format(os.getpid(), id(app), "fhost", id(get), escape(request.path)), e.code 436 437 @manager.command 438 def debug(): 439 app.config["FHOST_USE_X_ACCEL_REDIRECT"] = False 440 app.run(debug=True, port=4562,host="0.0.0.0") 441 442 @manager.command 443 def permadelete(name): 444 id = su.debase(name) 445 f = File.query.get(id) 446 447 if f: 448 if os.path.exists(getpath(f.sha256)): 449 os.remove(getpath(f.sha256)) 450 f.removed = True 451 db.session.commit() 452 453 @manager.command 454 def query(name): 455 id = su.debase(name) 456 f = File.query.get(id) 457 458 if f: 459 f.pprint() 460 461 @manager.command 462 def queryhash(h): 463 f = File.query.filter_by(sha256=h).first() 464 465 if f: 466 f.pprint() 467 468 @manager.command 469 def queryaddr(a, nsfw=False, removed=False): 470 res = File.query.filter_by(addr=a) 471 472 if not removed: 473 res = res.filter(File.removed != True) 474 475 if nsfw: 476 res = res.filter(File.nsfw_score > app.config["NSFW_THRESHOLD"]) 477 478 for f in res: 479 f.pprint() 480 481 @manager.command 482 def deladdr(a): 483 res = File.query.filter_by(addr=a).filter(File.removed != True) 484 485 for f in res: 486 if os.path.exists(getpath(f.sha256)): 487 os.remove(getpath(f.sha256)) 488 f.removed = True 489 490 db.session.commit() 491 492 def nsfw_detect(f): 493 try: 494 open(f["path"], 'r').close() 495 f["nsfw_score"] = nsfw.detect(f["path"]) 496 return f 497 except: 498 return None 499 500 @manager.command 501 def update_nsfw(): 502 if not app.config["NSFW_DETECT"]: 503 print("NSFW detection is disabled in app config") 504 return 1 505 506 from multiprocessing import Pool 507 import tqdm 508 509 res = File.query.filter_by(nsfw_score=None, removed=False) 510 511 with Pool() as p: 512 results = [] 513 work = [{ "path" : getpath(f.sha256), "id" : f.id} for f in res] 514 515 for r in tqdm.tqdm(p.imap_unordered(nsfw_detect, work), total=len(work)): 516 if r: 517 results.append({"id": r["id"], "nsfw_score" : r["nsfw_score"]}) 518 519 db.session.bulk_update_mappings(File, results) 520 db.session.commit() 521 522 523 @manager.command 524 def querybl(nsfw=False, removed=False): 525 blist = [] 526 if os.path.isfile(app.config["FHOST_UPLOAD_BLACKLIST"]): 527 with open(app.config["FHOST_UPLOAD_BLACKLIST"], "r") as bl: 528 for l in bl.readlines(): 529 if not l.startswith("#"): 530 if not ":" in l: 531 blist.append("::ffff:" + l.rstrip()) 532 else: 533 blist.append(l.strip()) 534 535 res = File.query.filter(File.addr.in_(blist)) 536 537 if not removed: 538 res = res.filter(File.removed != True) 539 540 if nsfw: 541 res = res.filter(File.nsfw_score > app.config["NSFW_THRESHOLD"]) 542 543 for f in res: 544 f.pprint() 545 546 if __name__ == "__main__": 547 manager.run()