Grotesque memory usage and leaks with flask-sqlalchemy on Debian Wheezy
I am making a simple web application that pulls (currently over 64000 fields in two tables and grows) data from a MySQL database using flask-sqlalchemy. I only ran one instance and did not notice any problems until I ran out of RAM and the thing stopped. I understand that
I might have to rethink how it all works since I can't keep adding RAM (it's a small VM, so I'm 512 to 768 MB, but still) ... but depressing Python should free up memory after processing the request ? Indeed, when running the same flash app on my Windows machine, it sucks up RAM (but only half as much!) And when it's done, it's freed up. Not so on a Debian machine that runs a lot of (supposedly) tiny applications. As far as I know, the lib versions running on both machines are the same. The Debian machine even has a later version of Python than the Windows machine. They both connect to the same database. How can I proceed?
from flask import Flask, request, jsonify
from flask.ext.sqlalchemy import SQLAlchemy
import re
from datetime import datetime
app = Flask(__name__)
app.config.from_pyfile('settings.cfg')
db = SQLAlchemy(app)
class Reports(db.Model):
__tablename__ = 'reports'
id = db.Column(db.Integer, primary_key=True)
ip = db.Column(db.Integer)
date = db.Column(db.DateTime)
sid = db.Column(db.Integer)
version = db.Column(db.Integer)
itemname = db.Column(db.String(25))
group = db.Column(db.Integer)
pclass = db.Column(db.String(15))
ltime = db.Column(db.Integer)
rlen = db.Column(db.Integer)
total = db.Column(db.Integer)
def __init__(self, pd):
self.date = datetime.utcnow()
self.sid = pd["sid"]
self.version = pd["version"]
self.itemname = pd["itemname"]
self.group = pd["group"]
self.pclass = pd["pclass"]
self.ltime = pd["ltime"]
self.rlen = pd["rlen"]
self.total = pd["total"]
class Perfdata(db.Model):
__tablename__ = 'perfdata'
reportid = db.Column(db.Integer, db.ForeignKey('reports.id'), primary_key=True)
l70 = db.Column(db.Integer)
l65 = db.Column(db.Integer)
l60 = db.Column(db.Integer)
l55 = db.Column(db.Integer)
l50 = db.Column(db.Integer)
l45 = db.Column(db.Integer)
l40 = db.Column(db.Integer)
l35 = db.Column(db.Integer)
l30 = db.Column(db.Integer)
def __init__(self, reportid, pd):
self.reportid = reportid
self.l70 = pd["l70"]
self.l65 = pd["l65"]
self.l60 = pd["l60"]
self.l55 = pd["l55"]
self.l50 = pd["l50"]
self.l45 = pd["l45"]
self.l40 = pd["l40"]
self.l35 = pd["l35"]
self.l30 = pd["l30"]
def buildlist(self):
plist = []
plist.append(self.l70)
plist.append(self.l65)
plist.append(self.l60)
plist.append(self.l55)
plist.append(self.l50)
plist.append(self.l45)
plist.append(self.l40)
plist.append(self.l35)
plist.append(self.l30)
return plist
@app.route('/ps', methods=['GET'])
def perfget():
response = []
for report, perf in db.session.query(Reports, Perfdata).all():
response.append("")
response.append("%s %s %s %s %s %s %s %s" % (report.version,
report.sid,
report.itemname,
report.group,
report.pclass,
report.ltime,
report.rlen,
report.total))
response.append("%s %s %s %s %s %s %s %s %s" % (perf.l70,
perf.l65,
perf.l60,
perf.l55,
perf.l50,
perf.l45,
perf.l40,
perf.l35,
perf.l30))
return '<br>\n'.join(response)
if __name__ == '__main__':
app.run()
source to share