def test_multiset_operations(self):
# Verify that adding a zero counter will strip zeros and negatives
c = Counter(a=10, b=-2, c=0) + Counter()
self.assertEqual(dict(c), dict(a=10))
elements = 'abcd'
for i in range(1000):
# test random pairs of multisets
p = Counter(dict((elem, randrange(-2,4)) for elem in elements))
p.update(e=1, f=-1, g=0)
q = Counter(dict((elem, randrange(-2,4)) for elem in elements))
q.update(h=1, i=-1, j=0)
for counterop, numberop in [
(Counter.__add__, lambda x, y: max(0, x+y)),
(Counter.__sub__, lambda x, y: max(0, x-y)),
(Counter.__or__, lambda x, y: max(0,x,y)),
(Counter.__and__, lambda x, y: max(0, min(x,y))),
]:
result = counterop(p, q)
for x in elements:
self.assertEqual(numberop(p[x], q[x]), result[x],
(counterop, x, p, q))
# verify that results exclude non-positive counts
self.assertTrue(x>0 for x in result.values())
elements = 'abcdef'
for i in range(100):
# verify that random multisets with no repeats are exactly like sets
p = Counter(dict((elem, randrange(0, 2)) for elem in elements))
q = Counter(dict((elem, randrange(0, 2)) for elem in elements))
for counterop, setop in [
(Counter.__sub__, set.__sub__),
(Counter.__or__, set.__or__),
(Counter.__and__, set.__and__),
]:
counter_result = counterop(p, q)
set_result = setop(set(p.elements()), set(q.elements()))
self.assertEqual(counter_result, dict.fromkeys(set_result, 1))
def plugin(srv, item):
srv.logging.debug("*** MODULE=%s: service=%s, target=%s", __file__, item.service, item.target)
# If the incoming payload has been transformed, use that,
# else the original payload
text = item.message
try:
# addrs is a list[] associated with a particular target.
# it can contain an arbitrary amount of entries that are just
# passed along to rrdtool
# mofified by otfdr @ github to accept abitray arguments with
# the payload and to not always add the 'N' in front
# 2017-06-05 - fix/enhancement for https://github.com/jpmens/mqttwarn/issues/248
if re.match( "^\d+$", text ):
rrdtool.update(item.addrs, "N:" + text)
else:
rrdtool.update(item.addrs + text.split())
except Exception as e:
srv.logging.warning("Cannot call rrdtool")
return False
return True
def update_data(self, entity, timestamp, name_data_map):
for table, data in name_data_map.items(): # for each section
try:
table += '.rrd'
table = table.replace('/', '_') # some stat name include /
entity_table = os.path.join(entity, table)
handle = self.get_handle(entity_table)
if handle:
handle.update(timestamp, data)
except Exception as e:
print(e)
print('on update table %s, %s (item: %d)' % (entity, table + '.rrd', len(data)))
print(data)
return True
def add_sample(self, timestamp_epoch, cpu_usage, mem_usage):
KOA_LOGGER.debug('[puller][sample] %s, %f, %f', self.dbname, cpu_usage, mem_usage)
try:
rrdtool.update(self.rrd_location, '%s:%s:%s' % (
timestamp_epoch,
round(cpu_usage, KOA_CONFIG.db_round_decimals),
round(mem_usage, KOA_CONFIG.db_round_decimals)))
except rrdtool.OperationalError as e:
KOA_LOGGER.error("failing adding rrd sample (%s)", e)
def click(x, y):
# draw dot
pos = t.position()
t.penup()
t.goto(x, y)
t.dot()
t.goto(*pos)
t.pendown()
# draw pattern
pattern.append((x, y))
if len(pattern) % 4 == 0:
save(pattern_path, pattern)
start, end, p1, p2 = pattern[-4:]
draw_bezier(t, bezier_steps, start, end, p1, p2)
turtle.update()
# set background image
# img = '/Users/gua/Desktop/heart.gif'
# turtle.bgpic(img)
def update(self, filename, time, value):
if value is None:
return
rrdtool.update(filename, "%d:%.4f" % (time, value))
def draw_pattern(pattern):
n = len(pattern) // 4
for i in range(n):
s, e = i * 4, i * 4 + 4
start, end, p1, p2 = pattern[s:e]
draw_bezier(t, bezier_steps, start, end, p1, p2)
turtle.update()
def update(self, *params):
result = ''
count = 0
#print(params)
if len(params) == 2 and isinstance(params[1], dict): # dict input
data = params[1]
keys = list(data.keys())
keys.sort()
result = '%d:' % params[0]
count = len(keys)
for key in keys:
#print('update>> ' + key)
result += '%d:' % data[key]
else:
count = len(params) - 1
for p in params:
result += '%d:' % p
result = result[0:-1]
#print ('## rrd update %s (%d): %s' % (self.filename, count, result))
#ret = rrdtool.update(self.filename, result)
try:
self.fifo.write('update %s %s\n' % (self.filename, result))
except Exception as e:
print('## pipe write excpetion')
print(e)
def update(self, tables, where, vars=None, _test=False, **values):
"""
Update `tables` with clause `where` (interpolated using `vars`)
and setting `values`.
>>> db = DB(None, {})
>>> name = 'Joseph'
>>> q = db.update('foo', where='name = $name', name='bob', age=2,
... created=SQLLiteral('NOW()'), vars=locals(), _test=True)
>>> q
<sql: "UPDATE foo SET age = 2, name = 'bob', created = NOW() WHERE name = 'Joseph'">
>>> q.query()
'UPDATE foo SET age = %s, name = %s, created = NOW() WHERE name = %s'
>>> q.values()
[2, 'bob', 'Joseph']
"""
if vars is None: vars = {}
where = self._where(where, vars)
query = (
"UPDATE " + sqllist(tables) +
" SET " + sqlwhere(values, ', ') +
" WHERE " + where)
if _test: return query
db_cursor = self._db_cursor()
self._db_execute(db_cursor, query)
if not self.ctx.transactions:
self.ctx.commit()
return db_cursor.rowcount
def show_particles(self, particles, show_frequency = 10):
turtle.shape('tri')
for i, particle in enumerate(particles):
if i % show_frequency == 0:
turtle.setposition((particle.x, particle.y))
turtle.setheading(90 - particle.heading)
turtle.color(self.weight_to_color(particle.weight))
turtle.stamp()
turtle.update()
def get_all_table_list(self, prefix): table_list = [] for entity in os.listdir(self.storage_path): entity_path = os.path.join(self.storage_path, entity) if os.path.isdir(entity_path): for table in os.listdir(entity_path): if table.startswith(prefix): table_list.append(entity + '/' + table) return table_list # update data use member functions, it makes leak because rrdupdate has it
def update(self, tables, where, vars=None, _test=False, **values):
"""
Update `tables` with clause `where` (interpolated using `vars`)
and setting `values`.
>>> db = DB(None, {})
>>> name = 'Joseph'
>>> q = db.update('foo', where='name = $name', name='bob', age=2,
... created=SQLLiteral('NOW()'), vars=locals(), _test=True)
>>> q
<sql: "UPDATE foo SET age = 2, name = 'bob', created = NOW() WHERE name = 'Joseph'">
>>> q.query()
'UPDATE foo SET age = %s, name = %s, created = NOW() WHERE name = %s'
>>> q.values()
[2, 'bob', 'Joseph']
"""
if vars is None: vars = {}
where = self._where(where, vars)
query = (
"UPDATE " + sqllist(tables) +
" SET " + sqlwhere(values, ', ') +
" WHERE " + where)
if _test: return query
db_cursor = self._db_cursor()
self._db_execute(db_cursor, query)
if not self.ctx.transactions:
self.ctx.commit()
return db_cursor.rowcount
def show_estimated_location(self, particles):
'''
Show average weighted mean location of the particles.
'''
x_accum = 0
y_accum = 0
heading_accum = 0
weight_accum = 0
num_particles = len(particles)
for particle in particles:
weight_accum += particle.weight
x_accum += particle.x * particle.weight
y_accum += particle.y * particle.weight
heading_accum += particle.heading * particle.weight
if weight_accum == 0:
return False
x_estimate = x_accum / weight_accum
y_estimate = y_accum / weight_accum
heading_estimate = heading_accum / weight_accum
turtle.color('orange')
turtle.setposition(x_estimate, y_estimate)
turtle.setheading(90 - heading_estimate)
turtle.shape('turtle')
turtle.stamp()
turtle.update()
def do_something(self):
print('### do something invoked')
start_timestamp = time.time()
count = 0
fifo = open(self.fifo_path, 'r')
buffer = ''
while True:
before = time.time()
tmp = fifo.read(4096)
after = time.time()
#print('>>> read: ', tmp)
length = len(tmp)
buffer += tmp
while True:
idx = buffer.find('\n')
if idx < 0:
break
line = buffer[:idx]
#print('>>> line: ', line)
buffer = buffer[idx+1:]
toks = line.split(' ', 2)
if toks[0] == 'update':
#print('update %s %s' % (toks[1], toks[2]))
try:
rrdtool.update(toks[1], toks[2])
except rrdtool.OperationalError as e:
#print(e)
continue
except Exception as e:
print(e)
continue
else:
pass
if (after - before) > 0.1:
time.sleep(1)
count += 1
if count > 50000:
print('quit by overcounting, running sec: %d' % (time.time() - start_timestamp))
sys.exit()
def update(self, tables, where, vars=None, _test=False, **values):
"""
Update `tables` with clause `where` (interpolated using `vars`)
and setting `values`.
>>> db = DB(None, {})
>>> name = 'Joseph'
>>> q = db.update('foo', where='name = $name', name='bob', age=2,
... created=SQLLiteral('NOW()'), vars=locals(), _test=True)
>>> q
<sql: "UPDATE foo SET age = 2, created = NOW(), name = 'bob' WHERE name = 'Joseph'">
>>> q.query()
'UPDATE foo SET age = %s, created = NOW(), name = %s WHERE name = %s'
>>> q.values()
[2, 'bob', 'Joseph']
"""
if vars is None: vars = {}
where = self._where(where, vars)
values = sorted(values.items(), key=lambda t: t[0])
query = (
"UPDATE " + sqllist(tables) +
" SET " + sqlwhere(values, ', ') +
" WHERE " + where)
if _test: return query
db_cursor = self._db_cursor()
self._db_execute(db_cursor, query)
if not self.ctx.transactions:
self.ctx.commit()
return db_cursor.rowcount
def show_robot(self, robot):
turtle.color('green')
turtle.shape('turtle')
turtle.shapesize(0.7, 0.7)
turtle.setposition((robot.x, robot.y))
turtle.setheading(90 - robot.heading)
turtle.stamp()
turtle.update()
def update(self, tables, where, vars=None, _test=False, **values):
"""
Update `tables` with clause `where` (interpolated using `vars`)
and setting `values`.
>>> db = DB(None, {})
>>> name = 'Joseph'
>>> q = db.update('foo', where='name = $name', name='bob', age=2,
... created=SQLLiteral('NOW()'), vars=locals(), _test=True)
>>> q
<sql: "UPDATE foo SET age = 2, name = 'bob', created = NOW() WHERE name = 'Joseph'">
>>> q.query()
'UPDATE foo SET age = %s, name = %s, created = NOW() WHERE name = %s'
>>> q.values()
[2, 'bob', 'Joseph']
"""
if vars is None: vars = {}
where = self._where(where, vars)
query = (
"UPDATE " + sqllist(tables) +
" SET " + sqlwhere(values, ', ') +
" WHERE " + where)
if _test: return query
db_cursor = self._db_cursor()
self._db_execute(db_cursor, query)
if not self.ctx.transactions:
self.ctx.commit()
return db_cursor.rowcount
def tick(self):
# Check the game state
# showsplash, running, gameover, paused
if self.state == "showsplash":
self.show_splash(self.splash_time)
elif self.state == "paused":
pass
elif self.state == "gameover":
pass
else:
# Iterate through all sprites and call their tick method
for sprite in Game.sprites:
if sprite.state:
sprite.tick()
# Iterate through all labels and call their update method
for label in Game.labels:
if label.text != "":
label.tick()
# Update the screen
self.update_screen()
def update(self, tables, where, vars=None, _test=False, **values):
"""
Update `tables` with clause `where` (interpolated using `vars`)
and setting `values`.
>>> db = DB(None, {})
>>> name = 'Joseph'
>>> q = db.update('foo', where='name = $name', name='bob', age=2,
... created=SQLLiteral('NOW()'), vars=locals(), _test=True)
>>> q
<sql: "UPDATE foo SET age = 2, name = 'bob', created = NOW() WHERE name = 'Joseph'">
>>> q.query()
'UPDATE foo SET age = %s, name = %s, created = NOW() WHERE name = %s'
>>> q.values()
[2, 'bob', 'Joseph']
"""
if vars is None: vars = {}
where = self._where(where, vars)
query = (
"UPDATE " + sqllist(tables) +
" SET " + sqlwhere(values, ', ') +
" WHERE " + where)
if _test: return query
db_cursor = self._db_cursor()
self._db_execute(db_cursor, query)
if not self.ctx.transactions:
self.ctx.commit()
return db_cursor.rowcount
def update_screen(self):
while time.time() < self.time + (1.0 / self.fps):
pass
turtle.update()
self.time = time.time()
def update(self, text):
self.text = text
self.tick()
def tick(self):
# Check the game state
# showsplash, running, gameover, paused
if self.state == "showsplash":
self.show_splash(self.splash_time)
elif self.state == "paused":
pass
elif self.state == "gameover":
pass
else:
# Iterate through all sprites and call their tick method
for sprite in Game.sprites:
if sprite.state:
sprite.tick()
# Iterate through all labels and call their update method
for label in Game.labels:
if label.text != "":
label.tick()
# Update the screen
self.update_screen()
def update_screen(self):
while time.time() < self.time + (1.0 / self.FPS):
pass
turtle.update()
self.time = time.time()
def tick(self):
# Check the game state
# showsplash, running, gameover, paused
if self.state == "showsplash":
self.show_splash(self.splash_time)
elif self.state == "paused":
pass
elif self.state == "gameover":
pass
else:
# Iterate through all sprites and call their tick method
for sprite in Game.sprites:
if sprite.state:
sprite.tick()
# Iterate through all labels and call their update method
for label in Game.labels:
if label.text != "":
label.tick()
# Update the screen
self.update_screen()
def update_screen(self):
while time.time() < self.time + (1.0 / self.FPS):
pass
turtle.update()
self.time = time.time()
def update(self, text):
self.text = text
self.tick()
def s(n, l):
if n == 0: # stop conditions
# draw filled rectangle
turtle.color('black')
turtle.begin_fill()
for _ in range (4):
turtle.forward(l)
turtle.left(90)
turtle.end_fill()
else: # recursion
# around center point create 8 smalles rectangles.
# create two rectangles on every side
# so you have to repeat it four times
for _ in range(4):
# first rectangle
s(n-1, l/3)
turtle.forward(l/3)
# second rectangle
s(n-1, l/3)
turtle.forward(l/3)
# go to next corner
turtle.forward(l/3)
turtle.left(90)
# update screen
turtle.update()
# --- main ---
# stop updating screen (to make it faster)
def show_maze(self):
turtle.setworldcoordinates(0, 0, self.width * 1.005, self.height * 1.005)
wally = turtle.Turtle()
wally.speed(0)
wally.width(1.5)
wally.hideturtle()
turtle.tracer(0, 0)
for i in range(self.num_rows):
for j in range(self.num_cols):
permissibilities = self.permissibilities(cell = (i,j))
turtle.up()
wally.setposition((j * self.grid_width, i * self.grid_height))
# Set turtle heading orientation
# 0 - east, 90 - north, 180 - west, 270 - south
wally.setheading(0)
if not permissibilities[0]:
wally.down()
else:
wally.up()
wally.forward(self.grid_width)
wally.setheading(90)
wally.up()
if not permissibilities[1]:
wally.down()
else:
wally.up()
wally.forward(self.grid_height)
wally.setheading(180)
wally.up()
if not permissibilities[2]:
wally.down()
else:
wally.up()
wally.forward(self.grid_width)
wally.setheading(270)
wally.up()
if not permissibilities[3]:
wally.down()
else:
wally.up()
wally.forward(self.grid_height)
wally.up()
turtle.update()
def patch_filters():
from jinja2.filters import FILTERS
from jinja2.asyncfilters import ASYNC_FILTERS
FILTERS.update(ASYNC_FILTERS)
def patch_filters():
from jinja2.filters import FILTERS
from jinja2.asyncfilters import ASYNC_FILTERS
FILTERS.update(ASYNC_FILTERS)
def patch_filters():
from jinja2.filters import FILTERS
from jinja2.asyncfilters import ASYNC_FILTERS
FILTERS.update(ASYNC_FILTERS)
def patch_filters():
from jinja2.filters import FILTERS
from jinja2.asyncfilters import ASYNC_FILTERS
FILTERS.update(ASYNC_FILTERS)
def log_user(bot: Bot, update: Update):
chat = update.effective_chat # type: Optional[Chat]
msg = update.effective_message # type: Optional[Message]
sql.update_user(msg.from_user.id,
msg.from_user.username,
chat.id,
chat.title)
if msg.reply_to_message:
sql.update_user(msg.reply_to_message.from_user.id,
msg.reply_to_message.from_user.username,
chat.id,
chat.title)
if msg.forward_from:
sql.update_user(msg.forward_from.id,
msg.forward_from.username)
def patch_filters():
from jinja2.filters import FILTERS
from jinja2.asyncfilters import ASYNC_FILTERS
FILTERS.update(ASYNC_FILTERS)
def forward_faq(update: Update, context: CallbackContext):
if update.message.chat.username not in [ONTOPIC_USERNAME, OFFTOPIC_USERNAME]:
return
admins = context.bot.get_chat_administrators(ONTOPIC_USERNAME)
if update.effective_user.id not in [x.user.id for x in admins]:
return
if not update.message:
return
reply_to = update.message.reply_to_message
if not reply_to:
return
try:
update.message.delete()
except BadRequest:
pass
# Forward message to FAQ channel
reply_to.forward(const.FAQ_CHANNEL_ID, disable_notification=True)
def patch_filters():
from jinja2.filters import FILTERS
from jinja2.asyncfilters import ASYNC_FILTERS
FILTERS.update(ASYNC_FILTERS)
def get_settings(bot: Bot, update: Update):
chat = update.effective_chat # type: Optional[Chat]
user = update.effective_user # type: Optional[User]
msg = update.effective_message # type: Optional[Message]
args = msg.text.split(None, 1)
# ONLY send settings in PM
if chat.type != chat.PRIVATE:
if is_user_admin(chat, user.id):
text = "Click here to get this chat's settings, as well as yours."
msg.reply_text(text,
reply_markup=InlineKeyboardMarkup(
[[InlineKeyboardButton(text="Settings",
url="t.me/{}?start=stngs_{}".format(
bot.username, chat.id))]]))
else:
text = "Click here to check your settings."
else:
send_settings(chat.id, user.id, True)
def patch_filters():
from jinja2.filters import FILTERS
from jinja2.asyncfilters import ASYNC_FILTERS
FILTERS.update(ASYNC_FILTERS)
def migrate_chats(bot: Bot, update: Update):
msg = update.effective_message # type: Optional[Message]
if msg.migrate_to_chat_id:
old_chat = update.effective_chat.id
new_chat = msg.migrate_to_chat_id
elif msg.migrate_from_chat_id:
old_chat = msg.migrate_from_chat_id
new_chat = update.effective_chat.id
else:
return
LOGGER.info("Migrating from %s, to %s", str(old_chat), str(new_chat))
for mod in MIGRATEABLE:
mod.__migrate__(old_chat, new_chat)
LOGGER.info("Successfully migrated!")
raise DispatcherHandlerStop
def patch_filters():
from jinja2.filters import FILTERS
from jinja2.asyncfilters import ASYNC_FILTERS
FILTERS.update(ASYNC_FILTERS)
def disable(bot: Bot, update: Update, args: List[str]):
chat = update.effective_chat # type: Optional[Chat]
if len(args) >= 1:
disable_cmd = args[0]
if disable_cmd.startswith(CMD_STARTERS):
disable_cmd = disable_cmd[1:]
if disable_cmd in set(DISABLE_CMDS + DISABLE_OTHER):
sql.disable_command(chat.id, disable_cmd)
update.effective_message.reply_text("Disabled the use of `{}`".format(disable_cmd),
parse_mode=ParseMode.MARKDOWN)
else:
update.effective_message.reply_text("That command can't be disabled")
else:
update.effective_message.reply_text("What should I disable?")
def patch_filters():
from jinja2.filters import FILTERS
from jinja2.asyncfilters import ASYNC_FILTERS
FILTERS.update(ASYNC_FILTERS)
def gbanlist(bot: Bot, update: Update):
banned_users = sql.get_gban_list()
if not banned_users:
update.effective_message.reply_text("There aren't any gbanned users! You're kinder than I expected...")
return
banfile = 'Screw these guys.\n'
for user in banned_users:
banfile += "[x] {} - {}\n".format(user["name"], user["user_id"])
if user["reason"]:
banfile += "Reason: {}\n".format(user["reason"])
with BytesIO(str.encode(banfile)) as output:
output.name = "gbanlist.txt"
update.effective_message.reply_document(document=output, filename="gbanlist.txt",
caption="Here is the list of currently gbanned users.")
def patch_filters():
from jinja2.filters import FILTERS
from jinja2.asyncfilters import ASYNC_FILTERS
FILTERS.update(ASYNC_FILTERS)
def enforce_gban(bot: Bot, update: Update):
# Not using @restrict handler to avoid spamming - just ignore if cant gban.
if sql.does_chat_gban(update.effective_chat.id) and update.effective_chat.get_member(bot.id).can_restrict_members:
user = update.effective_user # type: Optional[User]
chat = update.effective_chat # type: Optional[Chat]
msg = update.effective_message # type: Optional[Message]
if user and not is_user_admin(chat, user.id):
check_and_ban(update, user.id)
if msg.new_chat_members:
new_members = update.effective_message.new_chat_members
for mem in new_members:
check_and_ban(update, mem.id)
if msg.reply_to_message:
user = msg.reply_to_message.from_user # type: Optional[User]
if user and not is_user_admin(chat, user.id):
check_and_ban(update, user.id, should_message=False)
def patch_filters():
from jinja2.filters import FILTERS
from jinja2.asyncfilters import ASYNC_FILTERS
FILTERS.update(ASYNC_FILTERS)
def gbanstat(bot: Bot, update: Update, args: List[str]):
if len(args) > 0:
if args[0].lower() in ["on", "yes"]:
sql.enable_gbans(update.effective_chat.id)
update.effective_message.reply_text("I've enabled gbans in this group. This will help protect you "
"from spammers, unsavoury characters, and the biggest trolls.")
elif args[0].lower() in ["off", "no"]:
sql.disable_gbans(update.effective_chat.id)
update.effective_message.reply_text("I've disabled gbans in this group. GBans wont affect your users "
"anymore. You'll be less protected from any trolls and spammers "
"though!")
else:
update.effective_message.reply_text("Give me some arguments to choose a setting! on/off, yes/no!\n\n"
"Your current setting is: {}\n"
"When True, any gbans that happen will also happen in your group. "
"When False, they won't, leaving you at the possible mercy of "
"spammers.".format(sql.does_chat_gban(update.effective_chat.id)))
def patch_filters():
from jinja2.filters import FILTERS
from jinja2.asyncfilters import ASYNC_FILTERS
FILTERS.update(ASYNC_FILTERS)
def blacklist(bot: Bot, update: Update, args: List[str]):
msg = update.effective_message # type: Optional[Message]
chat = update.effective_chat # type: Optional[Chat]
all_blacklisted = sql.get_chat_blacklist(chat.id)
filter_list = BASE_BLACKLIST_STRING
if len(args) > 0 and args[0].lower() == 'copy':
for trigger in all_blacklisted:
filter_list += "<code>{}</code>\n".format(html.escape(trigger))
else:
for trigger in all_blacklisted:
filter_list += " - <code>{}</code>\n".format(html.escape(trigger))
split_text = split_message(filter_list)
for text in split_text:
if text == BASE_BLACKLIST_STRING:
msg.reply_text("There are no blacklisted messages here!")
return
msg.reply_text(text, parse_mode=ParseMode.HTML)
def patch_filters():
from jinja2.filters import FILTERS
from jinja2.asyncfilters import ASYNC_FILTERS
FILTERS.update(ASYNC_FILTERS)
def add_blacklist(bot: Bot, update: Update):
msg = update.effective_message # type: Optional[Message]
chat = update.effective_chat # type: Optional[Chat]
words = msg.text.split(None, 1)
if len(words) > 1:
text = words[1]
to_blacklist = list(set(trigger.strip() for trigger in text.split("\n") if trigger.strip()))
for trigger in to_blacklist:
sql.add_to_blacklist(chat.id, trigger.lower())
if len(to_blacklist) == 1:
msg.reply_text("Added <code>{}</code> to the blacklist!".format(html.escape(to_blacklist[0])),
parse_mode=ParseMode.HTML)
else:
msg.reply_text(
"Added <code>{}</code> triggers to the blacklist.".format(len(to_blacklist)), parse_mode=ParseMode.HTML)
else:
msg.reply_text("Tell me which words you would like to add to the blacklist.")
def patch_filters():
from jinja2.filters import FILTERS
from jinja2.asyncfilters import ASYNC_FILTERS
FILTERS.update(ASYNC_FILTERS)
def gmutelist(bot: Bot, update: Update):
muted_users = sql.get_gmute_list()
if not muted_users:
update.effective_message.reply_text("There aren't any gmuted users! You're kinder than I expected...")
return
mutefile = 'Screw these guys.\n'
for user in muted_users:
mutefile += "[x] {} - {}\n".format(user["name"], user["user_id"])
if user["reason"]:
mutefile += "Reason: {}\n".format(user["reason"])
with BytesIO(str.encode(mutefile)) as output:
output.name = "gmutelist.txt"
update.effective_message.reply_document(document=output, filename="gmutelist.txt",
caption="Here is the list of currently gmuted users.")
def patch_filters():
from jinja2.filters import FILTERS
from jinja2.asyncfilters import ASYNC_FILTERS
FILTERS.update(ASYNC_FILTERS)
def enforce_gmute(bot: Bot, update: Update):
# Not using @restrict handler to avoid spamming - just ignore if cant gmute.
if sql.does_chat_gmute(update.effective_chat.id) and update.effective_chat.get_member(bot.id).can_restrict_members:
user = update.effective_user # type: Optional[User]
chat = update.effective_chat # type: Optional[Chat]
msg = update.effective_message # type: Optional[Message]
if user and not is_user_admin(chat, user.id):
check_and_mute(bot, update, user.id, should_message=True)
if msg.new_chat_members:
new_members = update.effective_message.new_chat_members
for mem in new_members:
check_and_mute(bot, update, mem.id, should_message=True)
if msg.reply_to_message:
user = msg.reply_to_message.from_user # type: Optional[User]
if user and not is_user_admin(chat, user.id):
check_and_mute(bot, update, user.id, should_message=True)
def patch_filters():
from jinja2.filters import FILTERS
from jinja2.asyncfilters import ASYNC_FILTERS
FILTERS.update(ASYNC_FILTERS)
def list_handlers(bot: Bot, update: Update):
chat = update.effective_chat # type: Optional[Chat]
all_handlers = sql.get_chat_triggers(chat.id)
if not all_handlers:
update.effective_message.reply_text("No filters are active here!")
return
filter_list = BASIC_FILTER_STRING
for keyword in all_handlers:
entry = " - {}\n".format(escape_markdown(keyword))
if len(entry) + len(filter_list) > telegram.MAX_MESSAGE_LENGTH:
update.effective_message.reply_text(filter_list, parse_mode=telegram.ParseMode.MARKDOWN)
filter_list = entry
else:
filter_list += entry
if not filter_list == BASIC_FILTER_STRING:
update.effective_message.reply_text(filter_list, parse_mode=telegram.ParseMode.MARKDOWN)
# NOT ASYNC BECAUSE DISPATCHER HANDLER RAISED
def patch_filters():
from jinja2.filters import FILTERS
from jinja2.asyncfilters import ASYNC_FILTERS
FILTERS.update(ASYNC_FILTERS)
def stop_filter(bot: Bot, update: Update):
chat = update.effective_chat # type: Optional[Chat]
args = update.effective_message.text.split(None, 1)
if len(args) < 2:
return
chat_filters = sql.get_chat_triggers(chat.id)
if not chat_filters:
update.effective_message.reply_text("No filters are active here!")
return
for keyword in chat_filters:
if keyword == args[1]:
sql.remove_filter(chat.id, args[1])
update.effective_message.reply_text("Yep, I'll stop replying to that.")
raise DispatcherHandlerStop
update.effective_message.reply_text("That's not a current filter - run /filters for all active filters.")
def set_welcome(bot: Bot, update: Update) -> str:
chat = update.effective_chat # type: Optional[Chat]
user = update.effective_user # type: Optional[User]
msg = update.effective_message # type: Optional[Message]
text, data_type, content, buttons = get_welcome_type(msg)
if data_type is None:
msg.reply_text("You didn't specify what to reply with!")
return ""
sql.set_custom_welcome(chat.id, content or text, data_type, buttons)
msg.reply_text("Successfully set custom welcome message!")
return "<b>{}:</b>" \
"\n#SET_WELCOME" \
"\n<b>Admin:</b> {}" \
"\nSet the welcome message.".format(html.escape(chat.title),
mention_html(user.id, user.first_name))
def set_goodbye(bot: Bot, update: Update) -> str:
chat = update.effective_chat # type: Optional[Chat]
user = update.effective_user # type: Optional[User]
msg = update.effective_message # type: Optional[Message]
text, data_type, content, buttons = get_welcome_type(msg)
if data_type is None:
msg.reply_text("You didn't specify what to reply with!")
return ""
sql.set_custom_gdbye(chat.id, content or text, data_type, buttons)
msg.reply_text("Successfully set custom goodbye message!")
return "<b>{}:</b>" \
"\n#SET_GOODBYE" \
"\n<b>Admin:</b> {}" \
"\nSet the goodbye message.".format(html.escape(chat.title),
mention_html(user.id, user.first_name))
def about_me(bot: Bot, update: Update, args: List[str]):
message = update.effective_message # type: Optional[Message]
user_id = extract_user(message, args)
if user_id:
user = bot.get_chat(user_id)
else:
user = message.from_user
info = sql.get_user_me_info(user.id)
if info:
update.effective_message.reply_text("*{}*:\n{}".format(user.first_name, escape_markdown(info)),
parse_mode=ParseMode.MARKDOWN)
elif message.reply_to_message:
username = message.reply_to_message.from_user.first_name
update.effective_message.reply_text(username + " hasn't set an info message about themselves yet!")
else:
update.effective_message.reply_text("You haven't set an info message about yourself yet!")
def about_bio(bot: Bot, update: Update, args: List[str]):
message = update.effective_message # type: Optional[Message]
user_id = extract_user(message, args)
if user_id:
user = bot.get_chat(user_id)
else:
user = message.from_user
info = sql.get_user_bio(user.id)
if info:
update.effective_message.reply_text("*{}*:\n{}".format(user.first_name, escape_markdown(info)),
parse_mode=ParseMode.MARKDOWN)
elif message.reply_to_message:
username = user.first_name
update.effective_message.reply_text("{} hasn't had a message set about themselves yet!".format(username))
else:
update.effective_message.reply_text("You haven't had a bio set about yourself yet!")
def set_about_bio(bot: Bot, update: Update):
message = update.effective_message # type: Optional[Message]
sender = update.effective_user # type: Optional[User]
if message.reply_to_message:
repl_message = message.reply_to_message
user_id = repl_message.from_user.id
if user_id == message.from_user.id:
message.reply_text("Ha, you can't set your own bio! You're at the mercy of others here...")
return
elif user_id == bot.id and sender.id not in SUDO_USERS:
message.reply_text("Erm... yeah, I only trust sudo users to set my bio.")
return
text = message.text
bio = text.split(None, 1) # use python's maxsplit to only remove the cmd, hence keeping newlines.
if len(bio) == 2:
if len(bio[1]) < MAX_MESSAGE_LENGTH // 4:
sql.set_user_bio(user_id, bio[1])
message.reply_text("Updated {}'s bio!".format(repl_message.from_user.first_name))
else:
message.reply_text(
"A bio needs to be under {} characters! You tried to set {}.".format(
MAX_MESSAGE_LENGTH // 4, len(bio[1])))
else:
message.reply_text("Reply to someone's message to set their bio!")
def check_update(self, update):
if (isinstance(update, Update)
and (update.message or update.edited_message and self.allow_edited)):
message = update.message or update.edited_message
if message.text and len(message.text) > 1:
fst_word = message.text_html.split(None, 1)[0]
if len(fst_word) > 1 and any(fst_word.startswith(start) for start in CMD_STARTERS):
command = fst_word[1:].split('@')
command.append(message.bot.username) # in case the command was sent without a username
if self.filters is None:
res = True
elif isinstance(self.filters, list):
res = any(func(message) for func in self.filters)
else:
res = self.filters(message)
return res and (command[0].lower() in self.command
and command[1].lower() == message.bot.username.lower())
return False
def check_update(self, update):
if (isinstance(update, Update) and (update.message or update.edited_message and self.allow_edited)):
message = update.message or update.edited_message
if sql.is_user_gbanned(update.effective_user.id):
return False
if message.text and message.text.startswith('/') and len(message.text) > 1:
first_word = message.text_html.split(None, 1)[0]
if len(first_word) > 1 and first_word.startswith('/'):
command = first_word[1:].split('@')
command.append(message.bot.username) # in case the command was sent without a username
if not (command[0].lower() in self.command and command[1].lower() == message.bot.username.lower()):
return False
if self.filters is None:
res = True
elif isinstance(self.filters, list):
res = any(func(message) for func in self.filters)
else:
res = self.filters(message)
return res
return False
def user_admin(func):
@wraps(func)
def is_admin(bot: Bot, update: Update, *args, **kwargs):
user = update.effective_user # type: Optional[User]
if user and is_user_admin(update.effective_chat, user.id):
return func(bot, update, *args, **kwargs)
elif not user:
pass
elif DEL_CMDS and " " not in update.effective_message.text:
update.effective_message.delete()
else:
update.effective_message.reply_text("Who dis non-admin telling me what to do?")
return is_admin
def update(self, tables, where, vars=None, _test=False, **values):
"""
Update `tables` with clause `where` (interpolated using `vars`)
and setting `values`.
>>> db = DB(None, {})
>>> name = 'Joseph'
>>> q = db.update('foo', where='name = $name', name='bob', age=2,
... created=SQLLiteral('NOW()'), vars=locals(), _test=True)
>>> q
<sql: "UPDATE foo SET age = 2, name = 'bob', created = NOW() WHERE name = 'Joseph'">
>>> q.query()
'UPDATE foo SET age = %s, name = %s, created = NOW() WHERE name = %s'
>>> q.values()
[2, 'bob', 'Joseph']
"""
if vars is None: vars = {}
where = self._where(where, vars)
query = (
"UPDATE " + sqllist(tables) +
" SET " + sqlwhere(values, ', ') +
" WHERE " + where)
if _test: return query
db_cursor = self._db_cursor()
self._db_execute(db_cursor, query)
if not self.ctx.transactions:
self.ctx.commit()
return db_cursor.rowcount
def start(bot, update: Update, user: User, render):
update.message.reply_text(text=render('hello_message'))
def update(self, tables, where, vars=None, _test=False, **values):
"""
Update `tables` with clause `where` (interpolated using `vars`)
and setting `values`.
>>> db = DB(None, {})
>>> name = 'Joseph'
>>> q = db.update('foo', where='name = $name', name='bob', age=2,
... created=SQLLiteral('NOW()'), vars=locals(), _test=True)
>>> q
<sql: "UPDATE foo SET age = 2, name = 'bob', created = NOW() WHERE name = 'Joseph'">
>>> q.query()
'UPDATE foo SET age = %s, name = %s, created = NOW() WHERE name = %s'
>>> q.values()
[2, 'bob', 'Joseph']
"""
if vars is None: vars = {}
where = self._where(where, vars)
query = (
"UPDATE " + sqllist(tables) +
" SET " + sqlwhere(values, ', ') +
" WHERE " + where)
if _test: return query
db_cursor = self._db_cursor()
self._db_execute(db_cursor, query)
if not self.ctx.transactions:
self.ctx.commit()
return db_cursor.rowcount
def update(self, tables, where, vars=None, _test=False, **values):
"""
Update `tables` with clause `where` (interpolated using `vars`)
and setting `values`.
>>> db = DB(None, {})
>>> name = 'Joseph'
>>> q = db.update('foo', where='name = $name', name='bob', age=2,
... created=SQLLiteral('NOW()'), vars=locals(), _test=True)
>>> q
<sql: "UPDATE foo SET age = 2, name = 'bob', created = NOW() WHERE name = 'Joseph'">
>>> q.query()
'UPDATE foo SET age = %s, name = %s, created = NOW() WHERE name = %s'
>>> q.values()
[2, 'bob', 'Joseph']
"""
if vars is None: vars = {}
where = self._where(where, vars)
query = (
"UPDATE " + sqllist(tables) +
" SET " + sqlwhere(values, ', ') +
" WHERE " + where)
if _test: return query
db_cursor = self._db_cursor()
self._db_execute(db_cursor, query)
if not self.ctx.transactions:
self.ctx.commit()
return db_cursor.rowcount
def resend(bot, update: Update, user, render):
tasks.send_confirmation_mail.delay(user.pk)
update.message.reply_text(text=render('confirmation_message_is_sent'), reply_markup=ReplyKeyboardRemove())
def update(self, tables, where, vars=None, _test=False, **values):
"""
Update `tables` with clause `where` (interpolated using `vars`)
and setting `values`.
>>> db = DB(None, {})
>>> name = 'Joseph'
>>> q = db.update('foo', where='name = $name', name='bob', age=2,
... created=SQLLiteral('NOW()'), vars=locals(), _test=True)
>>> q
<sql: "UPDATE foo SET age = 2, name = 'bob', created = NOW() WHERE name = 'Joseph'">
>>> q.query()
'UPDATE foo SET age = %s, name = %s, created = NOW() WHERE name = %s'
>>> q.values()
[2, 'bob', 'Joseph']
"""
if vars is None: vars = {}
where = self._where(where, vars)
query = (
"UPDATE " + sqllist(tables) +
" SET " + sqlwhere(values, ', ') +
" WHERE " + where)
if _test: return query
db_cursor = self._db_cursor()
self._db_execute(db_cursor, query)
if not self.ctx.transactions:
self.ctx.commit()
return db_cursor.rowcount
def update(self, tables, where, vars=None, _test=False, **values):
"""
Update `tables` with clause `where` (interpolated using `vars`)
and setting `values`.
>>> db = DB(None, {})
>>> name = 'Joseph'
>>> q = db.update('foo', where='name = $name', name='bob', age=2,
... created=SQLLiteral('NOW()'), vars=locals(), _test=True)
>>> q
<sql: "UPDATE foo SET age = 2, created = NOW(), name = 'bob' WHERE name = 'Joseph'">
>>> q.query()
'UPDATE foo SET age = %s, created = NOW(), name = %s WHERE name = %s'
>>> q.values()
[2, 'bob', 'Joseph']
"""
if vars is None: vars = {}
where = self._where(where, vars)
values = sorted(values.items(), key=lambda t: t[0])
query = (
"UPDATE " + sqllist(tables) +
" SET " + sqlwhere(values, ', ') +
" WHERE " + where)
if _test: return query
db_cursor = self._db_cursor()
self._db_execute(db_cursor, query)
if not self.ctx.transactions:
self.ctx.commit()
return db_cursor.rowcount
def reset_email(bot, update: Update, user, render):
user.email = None
user.is_confirmed = False
user.save()
update.message.reply_text(text=render('email_is_reset'), reply_markup=ReplyKeyboardRemove())
def teardown_environment():
"""Restore things that were remembered by the setup_environment function
"""
orig_env = GIVEN_ENV['env']
for key in env.keys():
if key not in orig_env:
del env[key]
env.update(orig_env)
# decorator to use setup, teardown environment
def update(self, tables, where, vars=None, _test=False, **values):
"""
Update `tables` with clause `where` (interpolated using `vars`)
and setting `values`.
>>> db = DB(None, {})
>>> name = 'Joseph'
>>> q = db.update('foo', where='name = $name', name='bob', age=2,
... created=SQLLiteral('NOW()'), vars=locals(), _test=True)
>>> q
<sql: "UPDATE foo SET age = 2, created = NOW(), name = 'bob' WHERE name = 'Joseph'">
>>> q.query()
'UPDATE foo SET age = %s, created = NOW(), name = %s WHERE name = %s'
>>> q.values()
[2, 'bob', 'Joseph']
"""
if vars is None: vars = {}
where = self._where(where, vars)
values = sorted(values.items(), key=lambda t: t[0])
query = (
"UPDATE " + sqllist(tables) +
" SET " + sqlwhere(values, ', ') +
" WHERE " + where)
if _test: return query
db_cursor = self._db_cursor()
self._db_execute(db_cursor, query)
if not self.ctx.transactions:
self.ctx.commit()
return db_cursor.rowcount
def update(self, tables, where, vars=None, _test=False, **values):
"""
Update `tables` with clause `where` (interpolated using `vars`)
and setting `values`.
>>> db = DB(None, {})
>>> name = 'Joseph'
>>> q = db.update('foo', where='name = $name', name='bob', age=2,
... created=SQLLiteral('NOW()'), vars=locals(), _test=True)
>>> q
<sql: "UPDATE foo SET age = 2, name = 'bob', created = NOW() WHERE name = 'Joseph'">
>>> q.query()
'UPDATE foo SET age = %s, name = %s, created = NOW() WHERE name = %s'
>>> q.values()
[2, 'bob', 'Joseph']
"""
if vars is None: vars = {}
where = self._where(where, vars)
query = (
"UPDATE " + sqllist(tables) +
" SET " + sqlwhere(values, ', ') +
" WHERE " + where)
if _test: return query
db_cursor = self._db_cursor()
self._db_execute(db_cursor, query)
if not self.ctx.transactions:
self.ctx.commit()
return db_cursor.rowcount
def confirm_email(bot, update: Update, user, render):
key = update.message.text.strip()
if user.confirmation != key:
update.message.reply_text(text=render('confirmation_failure'))
return
user.is_confirmed = True
user.save()
update.message.reply_text(text=render('email_is_confirmed'))
def deploy_clojure(app, deltas={}):
"""Deploy a Clojure Application"""
virtual = join(ENV_ROOT, app)
target_path = join(APP_ROOT, app, 'target')
env_file = join(APP_ROOT, app, 'ENV')
if not exists(target_path):
makedirs(virtual)
env = {
'VIRTUAL_ENV': virtual,
"PATH": ':'.join([join(virtual, "bin"), join(app, ".bin"), environ['PATH']]),
"LEIN_HOME": environ.get('LEIN_HOME', join(environ['HOME'], '.lein')),
}
if exists(env_file):
env.update(parse_settings(env_file, env))
echo("-----> Building Clojure Application")
call('lein clean', cwd=join(APP_ROOT, app), env=env, shell=True)
call('lein uberjar', cwd=join(APP_ROOT, app), env=env, shell=True)
return spawn_app(app, deltas)
def update(self, tables, where, vars=None, _test=False, **values):
"""
Update `tables` with clause `where` (interpolated using `vars`)
and setting `values`.
>>> db = DB(None, {})
>>> name = 'Joseph'
>>> q = db.update('foo', where='name = $name', name='bob', age=2,
... created=SQLLiteral('NOW()'), vars=locals(), _test=True)
>>> q
<sql: "UPDATE foo SET age = 2, name = 'bob', created = NOW() WHERE name = 'Joseph'">
>>> q.query()
'UPDATE foo SET age = %s, name = %s, created = NOW() WHERE name = %s'
>>> q.values()
[2, 'bob', 'Joseph']
"""
if vars is None: vars = {}
where = self._where(where, vars)
query = (
"UPDATE " + sqllist(tables) +
" SET " + sqlwhere(values, ', ') +
" WHERE " + where)
if _test: return query
db_cursor = self._db_cursor()
self._db_execute(db_cursor, query)
if not self.ctx.transactions:
self.ctx.commit()
return db_cursor.rowcount
def update(self, tables, where, vars=None, _test=False, **values):
"""
Update `tables` with clause `where` (interpolated using `vars`)
and setting `values`.
>>> db = DB(None, {})
>>> name = 'Joseph'
>>> q = db.update('foo', where='name = $name', name='bob', age=2,
... created=SQLLiteral('NOW()'), vars=locals(), _test=True)
>>> q
<sql: "UPDATE foo SET age = 2, name = 'bob', created = NOW() WHERE name = 'Joseph'">
>>> q.query()
'UPDATE foo SET age = %s, name = %s, created = NOW() WHERE name = %s'
>>> q.values()
[2, 'bob', 'Joseph']
"""
if vars is None: vars = {}
where = self._where(where, vars)
query = (
"UPDATE " + sqllist(tables) +
" SET " + sqlwhere(values, ', ') +
" WHERE " + where)
if _test: return query
db_cursor = self._db_cursor()
self._db_execute(db_cursor, query)
if not self.ctx.transactions:
self.ctx.commit()
return db_cursor.rowcount
def send_text_message(bot, update: Update, user: User, render, **kwargs):
text = update.message.text
subject = get_subject(text)
update.message.reply_text(text=render('message_is_sent'))
tasks.send_text.delay(
user_id=user.pk,
subject=subject,
text=text,
)
def deploy_go(app, deltas={}):
"""Deploy a Go application"""
go_path = join(ENV_ROOT, app)
deps = join(APP_ROOT, app, 'Godeps')
first_time = False
if not exists(go_path):
echo("-----> Creating GOPATH for '{}'".format(app), fg='green')
makedirs(go_path)
# copy across a pre-built GOPATH to save provisioning time
call('cp -a $HOME/gopath {}'.format(app), cwd=ENV_ROOT, shell=True)
first_time = True
if exists(deps):
if first_time or getmtime(deps) > getmtime(go_path):
echo("-----> Running godep for '{}'".format(app), fg='green')
env = {
'GOPATH': '$HOME/gopath',
'GOROOT': '$HOME/go',
'PATH': '$PATH:$HOME/go/bin',
'GO15VENDOREXPERIMENT': '1'
}
call('godep update ...', cwd=join(APP_ROOT, app), env=env, shell=True)
return spawn_app(app, deltas)
def update(self, tables, where, vars=None, _test=False, **values):
"""
Update `tables` with clause `where` (interpolated using `vars`)
and setting `values`.
>>> db = DB(None, {})
>>> name = 'Joseph'
>>> q = db.update('foo', where='name = $name', name='bob', age=2,
... created=SQLLiteral('NOW()'), vars=locals(), _test=True)
>>> q
<sql: "UPDATE foo SET age = 2, name = 'bob', created = NOW() WHERE name = 'Joseph'">
>>> q.query()
'UPDATE foo SET age = %s, name = %s, created = NOW() WHERE name = %s'
>>> q.values()
[2, 'bob', 'Joseph']
"""
if vars is None: vars = {}
where = self._where(where, vars)
query = (
"UPDATE " + sqllist(tables) +
" SET " + sqlwhere(values, ', ') +
" WHERE " + where)
if _test: return query
db_cursor = self._db_cursor()
self._db_execute(db_cursor, query)
if not self.ctx.transactions:
self.ctx.commit()
return db_cursor.rowcount
def send_voice(bot, update: Update, user: User, render):
update.message.reply_text(text='Sorry, voice messages are not supported right now')
def cmd_git_receive_pack(app):
"""INTERNAL: Handle git pushes for an app"""
app = sanitize_app_name(app)
hook_path = join(GIT_ROOT, app, 'hooks', 'post-receive')
env = globals()
env.update(locals())
if not exists(hook_path):
makedirs(dirname(hook_path))
# Initialize the repository with a hook to this script
call("git init --quiet --bare " + app, cwd=GIT_ROOT, shell=True)
with open(hook_path, 'w') as h:
h.write("""#!/usr/bin/env bash
set -e; set -o pipefail;
cat | PIKU_ROOT="{PIKU_ROOT:s}" {PIKU_SCRIPT:s} git-hook {app:s}""".format(**env))
# Make the hook executable by our user
chmod(hook_path, stat(hook_path).st_mode | S_IXUSR)
# Handle the actual receive. We'll be called with 'git-hook' after it happens
call('git-shell -c "{}" '.format(argv[1] + " '{}'".format(app)), cwd=GIT_ROOT, shell=True)
def prompt_for_setting_email(bot, update: Update, user: User, render):
update.message.reply_text(text=render('please_send_email'))
def test_module(self):
environ.pop('DJANGO_ADMINS', None)
environ.update({
'DJANGO_DEBUG': 'true',
'DJANGO_INTERNAL_IPS': '127.0.0.1, 10.0.0.1',
'DJANGO_FILE_UPLOAD_DIRECTORY_PERMISSIONS': '0o644',
'DJANGO_SECRET_KEY': '123',
})
from . import settings_example as settings
self.assertEqual(global_settings.DEBUG, False)
self.assertEqual(settings.DEBUG, True)
if django.VERSION[:2] >= (1, 9):
self.assertListEqual(settings.INTERNAL_IPS, ['127.0.0.1', '10.0.0.1'])
else:
self.assertTupleEqual(settings.INTERNAL_IPS, ('127.0.0.1', '10.0.0.1'))
self.assertIsNone(global_settings.FILE_UPLOAD_DIRECTORY_PERMISSIONS)
self.assertEqual(settings.FILE_UPLOAD_DIRECTORY_PERMISSIONS, 420)
def send_confirmation(bot, update: Update, user: User, render):
email = update.message.text.strip()
if User.select().where(User.email == email):
update.message.reply_text(text=render('email_is_occupied'))
return
user.email = email
user.save()
tasks.send_confirmation_mail.delay(user.pk)
update.message.reply_text(text=render('confirmation_message_is_sent'))
def deploy_gradle(app, deltas={}):
"""Deploy a Java application using Gradle"""
java_path = join(ENV_ROOT, app)
build_path = join(APP_ROOT, app, 'build')
env_file = join(APP_ROOT, app, 'ENV')
env = {
'VIRTUAL_ENV': java_path,
"PATH": ':'.join([join(java_path, "bin"), join(app, ".bin"), environ['PATH']])
}
if exists(env_file):
env.update(parse_settings(env_file, env))
if not exists(java_path):
makedirs(java_path)
if not exists(build_path):
echo("-----> Building Java Application")
call('gradle build', cwd=join(APP_ROOT, app), env=env, shell=True)
else:
echo("-----> Removing previous builds")
echo("-----> Rebuilding Java Application")
call('gradle clean build', cwd=join(APP_ROOT, app), env=env, shell=True)
return spawn_app(app, deltas)
def deploy_java(app, deltas={}):
"""Deploy a Java application using Maven"""
# TODO: Use jenv to isolate Java Application environments
java_path = join(ENV_ROOT, app)
target_path = join(APP_ROOT, app, 'target')
env_file = join(APP_ROOT, app, 'ENV')
env = {
'VIRTUAL_ENV': java_path,
"PATH": ':'.join([join(java_path, "bin"), join(app, ".bin"), environ['PATH']])
}
if exists(env_file):
env.update(parse_settings(env_file, env))
if not exists(java_path):
makedirs(java_path)
if not exists(target_path):
echo("-----> Building Java Application")
call('mvn package', cwd=join(APP_ROOT, app), env=env, shell=True)
else:
echo("-----> Removing previous builds")
echo("-----> Rebuilding Java Application")
call('mvn clean package', cwd=join(APP_ROOT, app), env=env, shell=True)
return spawn_app(app, deltas)
def deploy_python(app, deltas={}):
"""Deploy a Python application"""
virtualenv_path = join(ENV_ROOT, app)
requirements = join(APP_ROOT, app, 'requirements.txt')
env_file = join(APP_ROOT, app, 'ENV')
# Peek at environment variables shipped with repo (if any) to determine version
env = {}
if exists(env_file):
env.update(parse_settings(env_file, env))
# TODO: improve version parsing
# pylint: disable=unused-variable
version = int(env.get("PYTHON_VERSION", "3"))
first_time = False
if not exists(virtualenv_path):
echo("-----> Creating virtualenv for '{}'".format(app), fg='green')
makedirs(virtualenv_path)
call('virtualenv --python=python{version:d} {app:s}'.format(**locals()), cwd=ENV_ROOT, shell=True)
first_time = True
activation_script = join(virtualenv_path, 'bin', 'activate_this.py')
exec(open(activation_script).read(), dict(__file__=activation_script))
if first_time or getmtime(requirements) > getmtime(virtualenv_path):
echo("-----> Running pip for '{}'".format(app), fg='green')
call('pip install -r {}'.format(requirements), cwd=virtualenv_path, shell=True)
return spawn_app(app, deltas)
def cmd_run(app, cmd):
"""e.g.: piku run <app> ls -- -al"""
app = exit_if_invalid(app)
config_file = join(ENV_ROOT, app, 'LIVE_ENV')
environ.update(parse_settings(config_file))
for f in [stdout, stderr]:
fl = fcntl(f, F_GETFL)
fcntl(f, F_SETFL, fl | O_NONBLOCK)
p = Popen(' '.join(cmd), stdin=stdin, stdout=stdout, stderr=stderr, env=environ, cwd=join(APP_ROOT, app), shell=True)
p.communicate()
def cmd_git_upload_pack(app):
"""INTERNAL: Handle git upload pack for an app"""
app = sanitize_app_name(app)
env = globals()
env.update(locals())
# Handle the actual receive. We'll be called with 'git-hook' after it happens
call('git-shell -c "{}" '.format(argv[1] + " '{}'".format(app)), cwd=GIT_ROOT, shell=True)
def use_matplot():
from matplotlib import use as mpl_use
# Fix possible problems with Mac OS X venv backend
if platform == "darwin":
mpl_use("TkAgg")
environ.update({"TK_SILENCE_DEPRECATION": "1"})
from matplotlib import pyplot as _plot
global plot
plot = _plot
def without_jira_vars():
backup = environ.copy()
del environ['JIRA_PROJECT']
del environ['JIRA_USER']
del environ['JIRA_PASSWORD']
del environ['JIRA_URL']
reload(jira)
yield
environ.update(backup)
reload(jira)
def temp_env(kw):
old = {k: environ.get(k) for k in kw}
environ.update(kw)
try:
yield
finally:
for key, value in old.items():
if value:
environ[key] = value
else:
del environ[key]
def params(cls):
""" Define configuration parameters.
"""
environ.update({"env1": "ENV1", "env2": "ENV2"})
return {"var1": "VAR1", "var2": "VAR2", "var3": "VAR3", "env2": "VAR2"}
def clean_state(self):
"""
Cleans up the state.
"""
if path.exists(self.state_file):
remove(self.state_file)
environ.update(self.old_environ)
def unset_git_env(self):
environ.update(self.old_environ)
def test_build_play_command_specified():
environ.update(FEDIPLAY_PLAY_COMMAND='afplay {filename}')
play_command = build_play_command('Awesome Music.mp3')
assert play_command == 'afplay \'Awesome Music.mp3\''
def do_deploy(app, deltas={}, newrev=None):
"""Deploy an app by resetting the work directory"""
app_path = join(APP_ROOT, app)
procfile = join(app_path, 'Procfile')
log_path = join(LOG_ROOT, app)
env = {'GIT_WORK_DIR': app_path}
if exists(app_path):
echo("-----> Deploying app '{}'".format(app), fg='green')
call('git fetch --quiet', cwd=app_path, env=env, shell=True)
if newrev:
call('git reset --hard {}'.format(newrev), cwd=app_path, env=env, shell=True)
call('git submodule init', cwd=app_path, env=env, shell=True)
call('git submodule update', cwd=app_path, env=env, shell=True)
if not exists(log_path):
makedirs(log_path)
workers = parse_procfile(procfile)
if workers and len(workers) > 0:
settings = {}
if exists(join(app_path, 'requirements.txt')) and found_app("Python"):
settings.update(deploy_python(app, deltas))
elif exists(join(app_path, 'package.json')) and found_app("Node") and (
check_requirements(['nodejs', 'npm']) or check_requirements(['nodeenv'])):
settings.update(deploy_node(app, deltas))
elif exists(join(app_path, 'pom.xml')) and found_app("Java Maven") and check_requirements(['java', 'mvn']):
settings.update(deploy_java(app, deltas))
elif exists(join(app_path, 'build.gradle')) and found_app("Java Gradle") and check_requirements(['java', 'gradle']):
settings.update(deploy_java(app, deltas))
elif (exists(join(app_path, 'Godeps')) or len(glob(join(app_path, '*.go')))) and found_app("Go") and check_requirements(['go']):
settings.update(deploy_go(app, deltas))
elif exists(join(app_path, 'project.clj')) and found_app("Clojure Lein") and check_requirements(['java', 'lein']):
settings.update(deploy_clojure(app, deltas))
elif 'release' in workers and 'web' in workers:
echo("-----> Generic app detected.", fg='green')
settings.update(deploy_identity(app, deltas))
elif 'static' in workers:
echo("-----> Static app detected.", fg='green')
settings.update(deploy_identity(app, deltas))
else:
echo("-----> Could not detect runtime!", fg='red')
# TODO: detect other runtimes
if "release" in workers:
echo("-----> Releasing", fg='green')
retval = call(workers["release"], cwd=app_path, env=settings, shell=True)
if retval:
echo("-----> Exiting due to release command error value: {}".format(retval))
exit(retval)
workers.pop("release", None)
else:
echo("Error: Invalid Procfile for app '{}'.".format(app), fg='red')
else:
echo("Error: app '{}' not found.".format(app), fg='red')
def formfield(self, **kwargs):
"""Passes ``limit_choices_to`` to field being constructed.
Only passes it if there is a type that supports related fields.
This is a similar strategy used to pass the ``queryset`` to the field
being constructed.
"""
defaults = {}
if hasattr(self.rel, 'get_related_field'):
# If this is a callable, do not invoke it here. Just pass
# it in the defaults for when the form class will later be
# instantiated.
limit_choices_to = self.rel.limit_choices_to
defaults.update({
'limit_choices_to': limit_choices_to,
})
defaults.update(kwargs)
return super(RelatedField, self).formfield(**defaults)
def __get__(self, instance, instance_type=None):
if instance is None:
return self
try:
rel_obj = getattr(instance, self.cache_name)
except AttributeError:
val = self.field.get_local_related_value(instance)
if None in val:
rel_obj = None
else:
params = {
rh_field.attname: getattr(instance, lh_field.attname)
for lh_field, rh_field in self.field.related_fields}
qs = self.get_queryset(instance=instance)
extra_filter = self.field.get_extra_descriptor_filter(instance)
if isinstance(extra_filter, dict):
params.update(extra_filter)
qs = qs.filter(**params)
else:
qs = qs.filter(extra_filter, **params)
# Assuming the database enforces foreign keys, this won't fail.
rel_obj = qs.get()
if not self.field.rel.multiple:
setattr(rel_obj, self.field.rel.get_cache_name(), instance)
setattr(instance, self.cache_name, rel_obj)
if rel_obj is None and not self.field.null:
raise self.RelatedObjectDoesNotExist(
"%s has no %s." % (self.field.model.__name__, self.field.name)
)
else:
return rel_obj
def formfield(self, **kwargs):
db = kwargs.pop('using', None)
if isinstance(self.rel.to, six.string_types):
raise ValueError("Cannot create form field for %r yet, because "
"its related model %r has not been loaded yet" %
(self.name, self.rel.to))
defaults = {
'form_class': forms.ModelChoiceField,
'queryset': self.rel.to._default_manager.using(db),
'to_field_name': self.rel.field_name,
}
defaults.update(kwargs)
return super(ForeignKey, self).formfield(**defaults)
def update(self, tables, where, vars=None, _test=False, **values):
"""
Update `tables` with clause `where` (interpolated using `vars`)
and setting `values`.
>>> db = DB(None, {})
>>> name = 'Joseph'
>>> q = db.update('foo', where='name = $name', name='bob', age=2,
... created=SQLLiteral('NOW()'), vars=locals(), _test=True)
>>> q
<sql: "UPDATE foo SET age = 2, name = 'bob', created = NOW() WHERE name = 'Joseph'">
>>> q.query()
'UPDATE foo SET age = %s, name = %s, created = NOW() WHERE name = %s'
>>> q.values()
[2, 'bob', 'Joseph']
"""
if vars is None: vars = {}
where = self._where(where, vars)
query = (
"UPDATE " + sqllist(tables) +
" SET " + sqlwhere(values, ', ') +
" WHERE " + where)
if _test: return query
db_cursor = self._db_cursor()
self._db_execute(db_cursor, query)
if not self.ctx.transactions:
self.ctx.commit()
return db_cursor.rowcount
def formfield(self, **kwargs):
db = kwargs.pop('using', None)
defaults = {
'form_class': forms.ModelMultipleChoiceField,
'queryset': self.rel.to._default_manager.using(db),
}
defaults.update(kwargs)
# If initial is passed in, it's a list of related objects, but the
# MultipleChoiceField takes a list of IDs.
if defaults.get('initial') is not None:
initial = defaults['initial']
if callable(initial):
initial = initial()
defaults['initial'] = [i._get_pk_val() for i in initial]
return super(ManyToManyField, self).formfield(**defaults)
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def update(self, tables, where, vars=None, _test=False, **values):
"""
Update `tables` with clause `where` (interpolated using `vars`)
and setting `values`.
>>> db = DB(None, {})
>>> name = 'Joseph'
>>> q = db.update('foo', where='name = $name', name='bob', age=2,
... created=SQLLiteral('NOW()'), vars=locals(), _test=True)
>>> q
<sql: "UPDATE foo SET age = 2, name = 'bob', created = NOW() WHERE name = 'Joseph'">
>>> q.query()
'UPDATE foo SET age = %s, name = %s, created = NOW() WHERE name = %s'
>>> q.values()
[2, 'bob', 'Joseph']
"""
if vars is None: vars = {}
where = self._where(where, vars)
query = (
"UPDATE " + sqllist(tables) +
" SET " + sqlwhere(values, ', ') +
" WHERE " + where)
if _test: return query
db_cursor = self._db_cursor()
self._db_execute(db_cursor, query)
if not self.ctx.transactions:
self.ctx.commit()
return db_cursor.rowcount
def __set__(self, instance, value):
super().__set__(instance, value)
# If the primary key is a link to a parent model and a parent instance
# is being set, update the value of the inherited pk(s).
if self.field.primary_key and self.field.remote_field.parent_link:
opts = instance._meta
# Inherited primary key fields from this object's base classes.
inherited_pk_fields = [
field for field in opts.concrete_fields
if field.primary_key and field.remote_field
]
for field in inherited_pk_fields:
rel_model_pk_name = field.remote_field.model._meta.pk.attname
raw_value = getattr(value, rel_model_pk_name) if value is not None else None
setattr(instance, rel_model_pk_name, raw_value)
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def update(self, tables, where, vars=None, _test=False, **values):
"""
Update `tables` with clause `where` (interpolated using `vars`)
and setting `values`.
>>> db = DB(None, {})
>>> name = 'Joseph'
>>> q = db.update('foo', where='name = $name', name='bob', age=2,
... created=SQLLiteral('NOW()'), vars=locals(), _test=True)
>>> q
<sql: "UPDATE foo SET age = 2, created = NOW(), name = 'bob' WHERE name = 'Joseph'">
>>> q.query()
'UPDATE foo SET age = %s, created = NOW(), name = %s WHERE name = %s'
>>> q.values()
[2, 'bob', 'Joseph']
"""
if vars is None: vars = {}
where = self._where(where, vars)
values = sorted(values.items(), key=lambda t: t[0])
query = (
"UPDATE " + sqllist(tables) +
" SET " + sqlwhere(values, ', ') +
" WHERE " + where)
if _test: return query
db_cursor = self._db_cursor()
self._db_execute(db_cursor, query)
if not self.ctx.transactions:
self.ctx.commit()
return db_cursor.rowcount
def __set__(self, instance, value):
super().__set__(instance, value)
# If the primary key is a link to a parent model and a parent instance
# is being set, update the value of the inherited pk(s).
if self.field.primary_key and self.field.remote_field.parent_link:
opts = instance._meta
# Inherited primary key fields from this object's base classes.
inherited_pk_fields = [
field for field in opts.concrete_fields
if field.primary_key and field.remote_field
]
for field in inherited_pk_fields:
rel_model_pk_name = field.remote_field.model._meta.pk.attname
raw_value = getattr(value, rel_model_pk_name) if value is not None else None
setattr(instance, rel_model_pk_name, raw_value)
def __init__(*args, **kwds):
'''Create a new, empty Counter object. And if given, count elements
from an input iterable. Or, initialize the count from another mapping
of elements to their counts.
>>> c = Counter() # a new, empty counter
>>> c = Counter('gallahad') # a new counter from an iterable
>>> c = Counter({'a': 4, 'b': 2}) # a new counter from a mapping
>>> c = Counter(a=4, b=2) # a new counter from keyword args
'''
if not args:
raise TypeError("descriptor '__init__' of 'Counter' object "
"needs an argument")
self = args[0]
args = args[1:]
if len(args) > 1:
raise TypeError('expected at most 1 arguments, got %d' % len(args))
super(Counter, self).__init__()
self.update(*args, **kwds)
def update(self, tables, where, vars=None, _test=False, **values):
"""
Update `tables` with clause `where` (interpolated using `vars`)
and setting `values`.
>>> db = DB(None, {})
>>> name = 'Joseph'
>>> q = db.update('foo', where='name = $name', name='bob', age=2,
... created=SQLLiteral('NOW()'), vars=locals(), _test=True)
>>> q
<sql: "UPDATE foo SET age = 2, name = 'bob', created = NOW() WHERE name = 'Joseph'">
>>> q.query()
'UPDATE foo SET age = %s, name = %s, created = NOW() WHERE name = %s'
>>> q.values()
[2, 'bob', 'Joseph']
"""
if vars is None: vars = {}
where = self._where(where, vars)
query = (
"UPDATE " + sqllist(tables) +
" SET " + sqlwhere(values, ', ') +
" WHERE " + where)
if _test: return query
db_cursor = self._db_cursor()
self._db_execute(db_cursor, query)
if not self.ctx.transactions:
self.ctx.commit()
return db_cursor.rowcount
def __init__(*args, **kwds):
'''Create a new, empty Counter object. And if given, count elements
from an input iterable. Or, initialize the count from another mapping
of elements to their counts.
>>> c = Counter() # a new, empty counter
>>> c = Counter('gallahad') # a new counter from an iterable
>>> c = Counter({'a': 4, 'b': 2}) # a new counter from a mapping
>>> c = Counter(a=4, b=2) # a new counter from keyword args
'''
if not args:
raise TypeError("descriptor '__init__' of 'Counter' object "
"needs an argument")
self = args[0]
args = args[1:]
if len(args) > 1:
raise TypeError('expected at most 1 arguments, got %d' % len(args))
super(Counter, self).__init__()
self.update(*args, **kwds)
def update(self, tables, where, vars=None, _test=False, **values):
"""
Update `tables` with clause `where` (interpolated using `vars`)
and setting `values`.
>>> db = DB(None, {})
>>> name = 'Joseph'
>>> q = db.update('foo', where='name = $name', name='bob', age=2,
... created=SQLLiteral('NOW()'), vars=locals(), _test=True)
>>> q
<sql: "UPDATE foo SET age = 2, name = 'bob', created = NOW() WHERE name = 'Joseph'">
>>> q.query()
'UPDATE foo SET age = %s, name = %s, created = NOW() WHERE name = %s'
>>> q.values()
[2, 'bob', 'Joseph']
"""
if vars is None: vars = {}
where = self._where(where, vars)
query = (
"UPDATE " + sqllist(tables) +
" SET " + sqlwhere(values, ', ') +
" WHERE " + where)
if _test: return query
db_cursor = self._db_cursor()
self._db_execute(db_cursor, query)
if not self.ctx.transactions:
self.ctx.commit()
return db_cursor.rowcount
def __init__(*args, **kwds):
'''Create a new, empty Counter object. And if given, count elements
from an input iterable. Or, initialize the count from another mapping
of elements to their counts.
>>> c = Counter() # a new, empty counter
>>> c = Counter('gallahad') # a new counter from an iterable
>>> c = Counter({'a': 4, 'b': 2}) # a new counter from a mapping
>>> c = Counter(a=4, b=2) # a new counter from keyword args
'''
if not args:
raise TypeError("descriptor '__init__' of 'Counter' object "
"needs an argument")
self = args[0]
args = args[1:]
if len(args) > 1:
raise TypeError('expected at most 1 arguments, got %d' % len(args))
super(Counter, self).__init__()
self.update(*args, **kwds)
def upgrade():
op.add_column('keypairs', sa.Column('ssh_public_key', sa.String(length=750), nullable=True))
op.add_column('keypairs', sa.Column('ssh_private_key', sa.String(length=2000), nullable=True))
# partial table to be preserved and referred
metadata = sa.MetaData(naming_convention=convention)
keypairs = sa.Table(
'keypairs', metadata,
sa.Column('access_key', sa.String(length=20), primary_key=True),
sa.Column('ssh_public_key', sa.String(length=750), nullable=True),
sa.Column('ssh_private_key', sa.String(length=2000), nullable=True),
)
# Fill in SSH keypairs in every keypairs.
conn = op.get_bind()
query = sa.select([keypairs.c.access_key]).select_from(keypairs)
rows = conn.execute(query).fetchall()
for row in rows:
pubkey, privkey = generate_ssh_keypair()
query = (sa.update(keypairs)
.values(ssh_public_key=pubkey, ssh_private_key=privkey)
.where(keypairs.c.access_key == row.access_key))
conn.execute(query)
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def recalc_concurrency_used(db_conn: SAConnection, access_key: AccessKey) -> None:
query = (
sa.update(keypairs)
.values(
concurrency_used=(
sa.select([sa.func.count(kernels.c.id)])
.select_from(kernels)
.where(
(kernels.c.access_key == access_key) &
(kernels.c.status.in_(USER_RESOURCE_OCCUPYING_KERNEL_STATUSES))
)
.as_scalar()
),
)
.where(keypairs.c.access_key == access_key)
)
await db_conn.execute(query)
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def _unreserve_agent_slots(
db_conn: SAConnection,
sess_ctx: PendingSession,
agent_ctx: AgentAllocationContext,
) -> None:
# Un-reserve agent slots, using a separate db txn.
query = (
sa.select([agents.c.occupied_slots], for_update=True)
.select_from(agents)
.where(agents.c.id == agent_ctx.agent_id))
current_occupied_slots = await db_conn.scalar(query)
query = (
sa.update(agents)
.values({
'occupied_slots': current_occupied_slots - sess_ctx.requested_slots
})
.where(agents.c.id == agent_ctx.agent_id))
await db_conn.execute(query)
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def set_kernel_status(self, kernel_id: KernelId,
status: KernelStatus,
reason: str = '', *,
db_conn: SAConnection = None):
assert status != KernelStatus.TERMINATED, \
'TERMINATED status update must be handled in ' \
'mark_kernel_terminated()'
data = {
'status': status,
'status_info': reason,
'status_changed': datetime.now(tzutc()),
}
async with reenter_txn(self.dbpool, db_conn) as conn:
query = (
sa.update(kernels)
.values(data)
.where(kernels.c.id == kernel_id)
)
await conn.execute(query)
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def update_invitation(request: web.Request, params: Any) -> web.Response:
'''
Update sent invitation's permission. Other fields are not allowed to be updated.
'''
dbpool = request.app['dbpool']
access_key = request['keypair']['access_key']
inv_id = request.match_info['inv_id']
perm = params['perm']
log.info('VFOLDER.UPDATE_INVITATION (ak:{}, inv:{})', access_key, inv_id)
async with dbpool.acquire() as conn:
query = (sa.update(vfolder_invitations)
.values(permission=perm)
.where(vfolder_invitations.c.id == inv_id)
.where(vfolder_invitations.c.inviter == request['user']['email'])
.where(vfolder_invitations.c.state == VFolderInvitationState.PENDING))
await conn.execute(query)
resp = {'msg': f'vfolder invitation updated: {inv_id}.'}
return web.json_response(resp, status=200)
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def delete(request: web.Request, params: Any) -> web.Response:
dbpool = request.app['dbpool']
template_id = request.match_info['template_id']
requester_access_key, owner_access_key = await get_access_key_scopes(request, params)
log.info('DELETE (ak:{0}/{1})',
requester_access_key, owner_access_key if owner_access_key != requester_access_key else '*')
async with dbpool.acquire() as conn, conn.begin():
query = (sa.select([session_templates.c.id])
.select_from(session_templates)
.where((session_templates.c.id == template_id) &
(session_templates.c.is_active)
))
result = await conn.scalar(query)
if not result:
raise TaskTemplateNotFound
query = (sa.update(session_templates)
.values(is_active=False)
.where((session_templates.c.id == template_id)))
result = await conn.execute(query)
assert result.rowcount == 1
return web.json_response({'success': True})
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def touch_collection_replicas(collection_replicas, session=None):
"""
Update the accessed_at timestamp of the given collection replicas.
:param collection_replicas: the list of collection replicas.
:param session: The database session in use.
:returns: True, if successful, False otherwise.
"""
now = datetime.utcnow()
for collection_replica in collection_replicas:
try:
session.query(models.CollectionReplica).filter_by(scope=collection_replica['scope'], name=collection_replica['name'], rse_id=collection_replica['rse_id']).\
update({'accessed_at': collection_replica.get('accessed_at') or now}, synchronize_session=False)
except DatabaseError:
return False
return True
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def set_tombstone(rse_id, scope, name, tombstone=OBSOLETE, session=None):
"""
Sets a tombstone on a replica.
:param rse_id: ID of RSE.
:param scope: scope of the replica DID.
:param name: name of the replica DID.
:param tombstone: the tombstone to set. Default is OBSOLETE
:param session: database session in use.
"""
stmt = update(models.RSEFileAssociation).where(and_(models.RSEFileAssociation.rse_id == rse_id, models.RSEFileAssociation.name == name, models.RSEFileAssociation.scope == scope,
~session.query(models.ReplicaLock).filter_by(scope=scope, name=name, rse_id=rse_id).exists()))\
.values(tombstone=tombstone)
result = session.execute(stmt)
if not result.rowcount:
try:
session.query(models.RSEFileAssociation).filter_by(scope=scope, name=name, rse_id=rse_id).one()
raise exception.ReplicaIsLocked('Replica %s:%s on RSE %s is locked.' % (scope, name, get_rse_name(rse_id=rse_id, session=session)))
except NoResultFound:
raise exception.ReplicaNotFound('Replica %s:%s on RSE %s could not be found.' % (scope, name, get_rse_name(rse_id=rse_id, session=session)))
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def set_requests_state(request_ids, new_state, session=None):
"""
Bulk update the state of requests. Fails silently if the request_id does not exist.
:param request_ids: List of (Request-ID as a 32 character hex string).
:param new_state: New state as string.
:param session: Database session to use.
"""
record_counter('core.request.set_requests_state')
try:
for request_id in request_ids:
set_request_state(request_id, new_state, session=session)
except IntegrityError as error:
raise RucioException(error.args)
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def touch_requests_by_rule(rule_id, session=None):
"""
Update the update time of requests in a rule. Fails silently if no requests on this rule.
:param rule_id: Rule-ID as a 32 character hex string.
:param session: Database session to use.
"""
record_counter('core.request.touch_requests_by_rule')
try:
session.query(models.Request).with_hint(models.Request, "INDEX(REQUESTS REQUESTS_RULEID_IDX)", 'oracle')\
.filter_by(rule_id=rule_id)\
.filter(models.Request.state.in_([RequestState.FAILED, RequestState.DONE, RequestState.LOST, RequestState.NO_SOURCES, RequestState.ONLY_TAPE_SOURCES]))\
.filter(models.Request.updated_at < datetime.datetime.utcnow())\
.update({'updated_at': datetime.datetime.utcnow() + datetime.timedelta(minutes=20)}, synchronize_session=False)
except IntegrityError as error:
raise RucioException(error.args)
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def release_waiting_requests_per_deadline(rse_id=None, deadline=1, session=None):
"""
Release waiting requests that were waiting too long and exceeded the maximum waiting time to be released.
If the DID of a request is attached to a dataset, the oldest requested_at date of all requests related to the dataset will be used for checking and all requests of this dataset will be released.
:param rse_id: The source RSE id.
:param deadline: Maximal waiting time in hours until a dataset gets released.
:param session: The database session.
"""
amount_released_requests = 0
if deadline:
grouped_requests_subquery, filtered_requests_subquery = create_base_query_grouped_fifo(rse_id, filter_by_rse='source', session=session)
old_requests_subquery = session.query(grouped_requests_subquery.c.name,
grouped_requests_subquery.c.scope,
grouped_requests_subquery.c.oldest_requested_at)\
.filter(grouped_requests_subquery.c.oldest_requested_at < datetime.datetime.now() - datetime.timedelta(hours=deadline))\
.subquery()
old_requests_subquery = session.query(filtered_requests_subquery.c.id)\
.join(old_requests_subquery, and_(filtered_requests_subquery.c.dataset_name == old_requests_subquery.c.name, filtered_requests_subquery.c.dataset_scope == old_requests_subquery.c.scope))
old_requests_subquery = old_requests_subquery.subquery()
statement = update(models.Request).where(models.Request.id.in_(old_requests_subquery)).values(state=RequestState.QUEUED)
amount_released_requests = session.execute(statement).rowcount
return amount_released_requests
def __init__(self, source=None, name=None, lookup=[], encoding='utf8', **settings):
""" Create a new template.
If the source parameter (str or buffer) is missing, the name argument
is used to guess a template filename. Subclasses can assume that
self.source and/or self.filename are set. Both are strings.
The lookup, encoding and settings parameters are stored as instance
variables.
The lookup parameter stores a list containing directory paths.
The encoding parameter should be used to decode byte strings or files.
The settings parameter contains a dict for engine-specific settings.
"""
self.name = name
self.source = source.read() if hasattr(source, 'read') else source
self.filename = source.filename if hasattr(source, 'filename') else None
self.lookup = [os.path.abspath(x) for x in lookup]
self.encoding = encoding
self.settings = self.settings.copy() # Copy from class variable
self.settings.update(settings) # Apply
if not self.source and self.name:
self.filename = self.search(self.name, self.lookup)
if not self.filename:
raise TemplateError('Template %s not found.' % repr(name))
if not self.source and not self.filename:
raise TemplateError('No template specified.')
self.prepare(**self.settings)
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def __touch_request(request_id, session=None):
"""
Update the timestamp of a request. Fails silently if the request_id does not exist.
:param request_id: Request-ID as a 32 character hex string.
:param session: Database session to use.
"""
record_counter('core.request.touch_request')
try:
rowcount = session.query(models.Request).filter_by(id=request_id).update({'updated_at': datetime.datetime.utcnow()}, synchronize_session=False)
except IntegrityError as error:
raise RucioException(error.args)
if not rowcount:
raise UnsupportedOperation("Request %s cannot be touched." % request_id)
def view(tpl_name, **defaults):
''' Decorator: renders a template for a handler.
The handler can control its behavior like that:
- return a dict of template vars to fill out the template
- return something other than a dict and the view decorator will not
process the template, but return the handler result as is.
This includes returning a HTTPResponse(dict) to get,
for instance, JSON with autojson or other castfilters.
'''
def decorator(func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
result = func(*args, **kwargs)
if isinstance(result, (dict, DictMixin)):
tplvars = defaults.copy()
tplvars.update(result)
return template(tpl_name, **tplvars)
elif result is None:
return template(tpl_name, defaults)
return result
return wrapper
return decorator
def __init__(self, iterable=None, **kwds):
'''Create a new, empty Counter object.
And if given, count elements from an input iterable. Or,
initialize the count from another mapping of elements to
their counts.
A new, empty counter:
>>> c = Counter()
A new counter from an iterable
>>> c = Counter('gallahad')
A new counter from a mapping
>>> c = Counter({'a': 4, 'b': 2})
A new counter from keyword args
>>> c = Counter(a=4, b=2)
'''
self.update(iterable, **kwds)
def __init__(self, table, name):
self.table = table
self.name = name
self.info = {}
if name is not None:
op = self.table.migration.operation
with cnx(self.table.migration) as conn:
columns = op.impl.dialect.get_columns(
conn, self.table.name, schema=table.schema)
for c in columns:
if c['name'] == name:
self.info.update(c)
if not self.info:
raise MigrationException(
"No column %r found on %r" % (name, self.table.name))
def __init__(self, source=None, name=None, lookup=[], encoding='utf8', **settings):
""" Create a new template.
If the source parameter (str or buffer) is missing, the name argument
is used to guess a template filename. Subclasses can assume that
self.source and/or self.filename are set. Both are strings.
The lookup, encoding and settings parameters are stored as instance
variables.
The lookup parameter stores a list containing directory paths.
The encoding parameter should be used to decode byte strings or files.
The settings parameter contains a dict for engine-specific settings.
"""
self.name = name
self.source = source.read() if hasattr(source, 'read') else source
self.filename = source.filename if hasattr(source, 'filename') else None
self.lookup = [os.path.abspath(x) for x in lookup]
self.encoding = encoding
self.settings = self.settings.copy() # Copy from class variable
self.settings.update(settings) # Apply
if not self.source and self.name:
self.filename = self.search(self.name, self.lookup)
if not self.filename:
raise TemplateError('Template %s not found.' % repr(name))
if not self.source and not self.filename:
raise TemplateError('No template specified.')
self.prepare(**self.settings)
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def reschedule(self, name):
state = yield from self.get(name)
if state.manual:
raise ValueError("Can't reschedule")
else:
local_offset = pytz.timezone(state.timezone).utcoffset(dt.datetime.utcnow())
cron = croniter(state.crontab, dt.datetime.utcnow() + local_offset)
reschedule = cron.get_next(dt.datetime) - local_offset
with (yield from self.db.engine) as conn:
yield from conn.execute(update(
Job.__table__
).where(
Job.name==name
).values(
active=True,
scheduled=reschedule,
))
def template(*args, **kwargs):
'''
Get a rendered template as a string iterator.
You can use a name, a filename or a template string as first parameter.
Template rendering arguments can be passed as dictionaries
or directly (as keyword arguments).
'''
tpl = args[0] if args else None
adapter = kwargs.pop('template_adapter', SimpleTemplate)
lookup = kwargs.pop('template_lookup', TEMPLATE_PATH)
tplid = (id(lookup), tpl)
if tplid not in TEMPLATES or DEBUG:
settings = kwargs.pop('template_settings', {})
if isinstance(tpl, adapter):
TEMPLATES[tplid] = tpl
if settings: TEMPLATES[tplid].prepare(**settings)
elif "\n" in tpl or "{" in tpl or "%" in tpl or '$' in tpl:
TEMPLATES[tplid] = adapter(source=tpl, lookup=lookup, **settings)
else:
TEMPLATES[tplid] = adapter(name=tpl, lookup=lookup, **settings)
if not TEMPLATES[tplid]:
abort(500, 'Template (%s) not found' % tpl)
for dictarg in args[1:]: kwargs.update(dictarg)
return TEMPLATES[tplid].render(kwargs)
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def take(self, name):
state = yield from self.get(name)
if state.active == True:
raise ValueError("In progress already")
with (yield from self.db.engine) as conn:
if state.manual is False:
yield from self.reschedule(name)
result = yield from conn.execute(update(
Job.__table__
).where(
Job.name==name
).values(
active=True
))
def view(tpl_name, **defaults):
''' Decorator: renders a template for a handler.
The handler can control its behavior like that:
- return a dict of template vars to fill out the template
- return something other than a dict and the view decorator will not
process the template, but return the handler result as is.
This includes returning a HTTPResponse(dict) to get,
for instance, JSON with autojson or other castfilters.
'''
def decorator(func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
result = func(*args, **kwargs)
if isinstance(result, (dict, DictMixin)):
tplvars = defaults.copy()
tplvars.update(result)
return template(tpl_name, **tplvars)
return result
return wrapper
return decorator
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def read_only_read_sa_transaction(conn, deferrable):
await conn.execute(sa.insert(users).values(id=1, name='test1'))
t1 = await conn.begin(
isolation_level='SERIALIZABLE',
readonly=True,
deferrable=deferrable
)
where = users.c.id == 1
try:
await conn.execute(sa.update(users).values({'name': 't'}).where(where))
except InternalError as e:
assert e.pgcode == '25006'
await t1.commit()
await conn.execute(sa.delete(users))
assert len(await (await conn.execute(users.select())).fetchall()) == 0
def __init__(self, source=None, name=None, lookup=[], encoding='utf8', **settings):
""" Create a new template.
If the source parameter (str or buffer) is missing, the name argument
is used to guess a template filename. Subclasses can assume that
self.source and/or self.filename are set. Both are strings.
The lookup, encoding and settings parameters are stored as instance
variables.
The lookup parameter stores a list containing directory paths.
The encoding parameter should be used to decode byte strings or files.
The settings parameter contains a dict for engine-specific settings.
"""
self.name = name
self.source = source.read() if hasattr(source, 'read') else source
self.filename = source.filename if hasattr(source, 'filename') else None
self.lookup = map(os.path.abspath, lookup)
self.encoding = encoding
self.settings = self.settings.copy() # Copy from class variable
self.settings.update(settings) # Apply
if not self.source and self.name:
self.filename = self.search(self.name, self.lookup)
if not self.filename:
raise TemplateError('Template %s not found.' % repr(name))
if not self.source and not self.filename:
raise TemplateError('No template specified.')
self.prepare(**self.settings)
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def isolation_read_sa_transaction(conn, conn2):
await conn.execute(sa.insert(users).values(id=1, name='test1'))
t1 = await conn.begin()
where = users.c.id == 1
q_user = users.select().where(where)
user = await (await conn.execute(q_user)).fetchone()
assert await (await conn2.execute(q_user)).fetchone() == user
await conn.execute(sa.update(users).values({'name': 'name2'}).where(where))
t2 = await conn2.begin()
assert await (await conn2.execute(q_user)).fetchone() == user
await t1.commit()
await conn2.execute(sa.update(users).values(user).where(where))
await t2.commit()
assert await (await conn2.execute(q_user)).fetchone() == user
await conn.execute(sa.delete(users))
assert len(await (await conn.execute(users.select())).fetchall()) == 0
def template(*args, **kwargs):
'''
Get a rendered template as a string iterator.
You can use a name, a filename or a template string as first parameter.
Template rendering arguments can be passed as dictionaries
or directly (as keyword arguments).
'''
tpl = args[0] if args else None
template_adapter = kwargs.pop('template_adapter', SimpleTemplate)
if tpl not in TEMPLATES or DEBUG:
settings = kwargs.pop('template_settings', {})
lookup = kwargs.pop('template_lookup', TEMPLATE_PATH)
if isinstance(tpl, template_adapter):
TEMPLATES[tpl] = tpl
if settings: TEMPLATES[tpl].prepare(**settings)
elif "\n" in tpl or "{" in tpl or "%" in tpl or '$' in tpl:
TEMPLATES[tpl] = template_adapter(source=tpl, lookup=lookup, **settings)
else:
TEMPLATES[tpl] = template_adapter(name=tpl, lookup=lookup, **settings)
if not TEMPLATES[tpl]:
abort(500, 'Template (%s) not found' % tpl)
for dictarg in args[1:]: kwargs.update(dictarg)
return TEMPLATES[tpl].render(kwargs)
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def inline(self):
"""Make this :class:`_expression.Update` construct "inline" .
When set, SQL defaults present on :class:`_schema.Column`
objects via the
``default`` keyword will be compiled 'inline' into the statement and
not pre-executed. This means that their values will not be available
in the dictionary returned from
:meth:`_engine.CursorResult.last_updated_params`.
.. versionchanged:: 1.4 the :paramref:`_expression.update.inline`
parameter
is now superseded by the :meth:`_expression.Update.inline` method.
"""
self._inline = True
def view(tpl_name, **defaults):
''' Decorator: renders a template for a handler.
The handler can control its behavior like that:
- return a dict of template vars to fill out the template
- return something other than a dict and the view decorator will not
process the template, but return the handler result as is.
This includes returning a HTTPResponse(dict) to get,
for instance, JSON with autojson or other castfilters.
'''
def decorator(func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
result = func(*args, **kwargs)
if isinstance(result, (dict, DictMixin)):
tplvars = defaults.copy()
tplvars.update(result)
return template(tpl_name, **tplvars)
return result
return wrapper
return decorator
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def _reserve_agent(
sched_ctx: SchedulingContext,
db_conn: SAConnection,
scaling_group: str,
agent_id: AgentId,
requested_slots: ResourceSlot,
) -> AgentAllocationContext:
query = (
sa.select([agents.c.occupied_slots], for_update=True)
.select_from(agents)
.where(agents.c.id == agent_id))
current_occupied_slots = await db_conn.scalar(query)
query = (sa.update(agents)
.values({
'occupied_slots': current_occupied_slots + requested_slots
})
.where(agents.c.id == agent_id))
await db_conn.execute(query)
# Get the agent address for later RPC calls
query = (sa.select([agents.c.addr])
.where(agents.c.id == agent_id))
agent_addr = await db_conn.scalar(query)
assert agent_addr is not None
return AgentAllocationContext(agent_id, agent_addr, scaling_group)
def execute(self, _stdout, kwargs):
env = self.defaults.copy()
env.update(kwargs)
env.update({
'_stdout': _stdout,
'_printlist': _stdout.extend,
'include': functools.partial(self._include, env),
'rebase': functools.partial(self._rebase, env),
'_rebase': None,
'_str': self._str,
'_escape': self._escape,
'get': env.get,
'setdefault': env.setdefault,
'defined': env.__contains__
})
exec(self.co, env)
if env.get('_rebase'):
subtpl, rargs = env.pop('_rebase')
rargs['base'] = ''.join(_stdout) #copy stdout
del _stdout[:] # clear stdout
return self._include(env, subtpl, **rargs)
return env
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def check_keypair_resource_limit(
db_conn: SAConnection,
sched_ctx: SchedulingContext,
sess_ctx: PendingSession,
) -> PredicateResult:
query = (
sa.select([keypair_resource_policies])
.select_from(keypair_resource_policies)
.where(keypair_resource_policies.c.name == sess_ctx.resource_policy)
)
result = await db_conn.execute(query)
resource_policy = await result.first()
total_keypair_allowed = ResourceSlot.from_policy(resource_policy,
sched_ctx.known_slot_types)
key_occupied = await sched_ctx.registry.get_keypair_occupancy(
sess_ctx.access_key, conn=db_conn)
log.debug('keypair:{} current-occupancy: {}', sess_ctx.access_key, key_occupied)
log.debug('keypair:{} total-allowed: {}', sess_ctx.access_key, total_keypair_allowed)
if not (key_occupied + sess_ctx.requested_slots <= total_keypair_allowed):
async def update_status_info(
db_conn: SAConnection,
sched_ctx: SchedulingContext,
sess_ctx: PendingSession,
) -> None:
query = (sa.update(kernels)
.values(status_info='out-of-resource (keypair resource quota exceeded)')
.where(kernels.c.id == sess_ctx.kernel_id))
await db_conn.execute(query)
return PredicateResult(
False,
'Your keypair resource quota is exceeded. ({})'
.format(' '.join(
f'{k}={v}' for k, v in
total_keypair_allowed.to_humanized(sched_ctx.known_slot_types).items()
)),
failure_cb=update_status_info)
return PredicateResult(True)
def template(*args, **kwargs):
"""
Get a rendered template as a string iterator.
You can use a name, a filename or a template string as first parameter.
Template rendering arguments can be passed as dictionaries
or directly (as keyword arguments).
"""
tpl = args[0] if args else None
for dictarg in args[1:]:
kwargs.update(dictarg)
adapter = kwargs.pop('template_adapter', SimpleTemplate)
lookup = kwargs.pop('template_lookup', TEMPLATE_PATH)
tplid = (id(lookup), tpl)
if tplid not in TEMPLATES or DEBUG:
settings = kwargs.pop('template_settings', {})
if isinstance(tpl, adapter):
TEMPLATES[tplid] = tpl
if settings: TEMPLATES[tplid].prepare(**settings)
elif "\n" in tpl or "{" in tpl or "%" in tpl or '$' in tpl:
TEMPLATES[tplid] = adapter(source=tpl, lookup=lookup, **settings)
else:
TEMPLATES[tplid] = adapter(name=tpl, lookup=lookup, **settings)
if not TEMPLATES[tplid]:
abort(500, 'Template (%s) not found' % tpl)
return TEMPLATES[tplid].render(kwargs)
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def check_group_resource_limit(
db_conn: SAConnection,
sched_ctx: SchedulingContext,
sess_ctx: PendingSession,
) -> PredicateResult:
query = (sa.select([groups.c.total_resource_slots])
.where(groups.c.id == sess_ctx.group_id))
group_resource_slots = await db_conn.scalar(query)
group_resource_policy = {'total_resource_slots': group_resource_slots,
'default_for_unspecified': DefaultForUnspecified.UNLIMITED}
total_group_allowed = ResourceSlot.from_policy(group_resource_policy,
sched_ctx.known_slot_types)
group_occupied = await sched_ctx.registry.get_group_occupancy(
sess_ctx.group_id, conn=db_conn)
log.debug('group:{} current-occupancy: {}', sess_ctx.group_id, group_occupied)
log.debug('group:{} total-allowed: {}', sess_ctx.group_id, total_group_allowed)
if not (group_occupied + sess_ctx.requested_slots <= total_group_allowed):
async def update_status_info(
db_conn: SAConnection,
sched_ctx: SchedulingContext,
sess_ctx: PendingSession,
) -> None:
query = (sa.update(kernels)
.values(status_info='out-of-resource (group resource quota exceeded)')
.where(kernels.c.id == sess_ctx.kernel_id))
await db_conn.execute(query)
return PredicateResult(
False,
'Your group resource quota is exceeded. ({})'
.format(' '.join(
f'{k}={v}' for k, v in
total_group_allowed.to_humanized(sched_ctx.known_slot_types).items()
)),
failure_cb=update_status_info)
return PredicateResult(True)
def __init__(self, source=None, name=None, lookup=[], encoding='utf8', **settings):
""" Create a new template.
If the source parameter (str or buffer) is missing, the name argument
is used to guess a template filename. Subclasses can assume that
self.source and/or self.filename are set. Both are strings.
The lookup, encoding and settings parameters are stored as instance
variables.
The lookup parameter stores a list containing directory paths.
The encoding parameter should be used to decode byte strings or files.
The settings parameter contains a dict for engine-specific settings.
"""
self.name = name
self.source = source.read() if hasattr(source, 'read') else source
self.filename = source.filename if hasattr(source, 'filename') else None
self.lookup = [os.path.abspath(x) for x in lookup]
self.encoding = encoding
self.settings = self.settings.copy() # Copy from class variable
self.settings.update(settings) # Apply
if not self.source and self.name:
self.filename = self.search(self.name, self.lookup)
if not self.filename:
raise TemplateError('Template %s not found.' % repr(name))
if not self.source and not self.filename:
raise TemplateError('No template specified.')
self.prepare(**self.settings)
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def check_domain_resource_limit(
db_conn: SAConnection,
sched_ctx: SchedulingContext,
sess_ctx: PendingSession,
) -> PredicateResult:
query = (sa.select([domains.c.total_resource_slots])
.where(domains.c.name == sess_ctx.domain_name))
domain_resource_slots = await db_conn.scalar(query)
domain_resource_policy = {
'total_resource_slots': domain_resource_slots,
'default_for_unspecified': DefaultForUnspecified.UNLIMITED
}
total_domain_allowed = ResourceSlot.from_policy(domain_resource_policy,
sched_ctx.known_slot_types)
domain_occupied = await sched_ctx.registry.get_domain_occupancy(
sess_ctx.domain_name, conn=db_conn)
log.debug('domain:{} current-occupancy: {}', sess_ctx.domain_name, domain_occupied)
log.debug('domain:{} total-allowed: {}', sess_ctx.domain_name, total_domain_allowed)
if not (domain_occupied + sess_ctx.requested_slots <= total_domain_allowed):
async def update_status_info(
db_conn: SAConnection,
sched_ctx: SchedulingContext,
sess_ctx: PendingSession,
) -> None:
query = (sa.update(kernels)
.values(status_info='out-of-resource (domain resource quota exceeded)')
.where(kernels.c.id == sess_ctx.kernel_id))
await db_conn.execute(query)
return PredicateResult(
False,
'Your domain resource quota is exceeded. ({})'
.format(' '.join(
f'{k}={v}' for k, v in
total_domain_allowed.to_humanized(sched_ctx.known_slot_types).items()
)),
failure_cb=update_status_info)
return PredicateResult(True)
def template(*args, **kwargs):
'''
Get a rendered template as a string iterator.
You can use a name, a filename or a template string as first parameter.
Template rendering arguments can be passed as dictionaries
or directly (as keyword arguments).
'''
tpl = args[0] if args else None
adapter = kwargs.pop('template_adapter', SimpleTemplate)
lookup = kwargs.pop('template_lookup', TEMPLATE_PATH)
tplid = (id(lookup), tpl)
if tplid not in TEMPLATES or DEBUG:
settings = kwargs.pop('template_settings', {})
if isinstance(tpl, adapter):
TEMPLATES[tplid] = tpl
if settings: TEMPLATES[tplid].prepare(**settings)
elif "\n" in tpl or "{" in tpl or "%" in tpl or '$' in tpl:
TEMPLATES[tplid] = adapter(source=tpl, lookup=lookup, **settings)
else:
TEMPLATES[tplid] = adapter(name=tpl, lookup=lookup, **settings)
if not TEMPLATES[tplid]:
abort(500, 'Template (%s) not found' % tpl)
for dictarg in args[1:]: kwargs.update(dictarg)
return TEMPLATES[tplid].render(kwargs)
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def update_instance(self, inst_id, updated_fields):
async with self.dbpool.acquire() as conn, conn.begin():
query = (sa.update(agents)
.values(**updated_fields)
.where(agents.c.id == inst_id))
await conn.execute(query)
def view(tpl_name, **defaults):
''' Decorator: renders a template for a handler.
The handler can control its behavior like that:
- return a dict of template vars to fill out the template
- return something other than a dict and the view decorator will not
process the template, but return the handler result as is.
This includes returning a HTTPResponse(dict) to get,
for instance, JSON with autojson or other castfilters.
'''
def decorator(func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
result = func(*args, **kwargs)
if isinstance(result, (dict, DictMixin)):
tplvars = defaults.copy()
tplvars.update(result)
return template(tpl_name, **tplvars)
elif result is None:
return template(tpl_name, defaults)
return result
return wrapper
return decorator
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def increment_session_usage(self, sess_id, access_key, conn=None):
async with reenter_txn(self.dbpool, conn) as conn:
query = (sa.update(kernels)
.values(num_queries=kernels.c.num_queries + 1)
.where((kernels.c.sess_id == sess_id) &
(kernels.c.access_key == access_key) &
(kernels.c.role == 'master')))
await conn.execute(query)
def __init__(self, name, impmask):
''' Create a virtual package that redirects imports (see PEP 302). '''
self.name = name
self.impmask = impmask
self.module = sys.modules.setdefault(name, imp.new_module(name))
self.module.__dict__.update({'__file__': __file__, '__path__': [],
'__all__': [], '__loader__': self})
sys.meta_path.append(self)
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def set_session_status(
self,
session_id: SessionId,
access_key: AccessKey,
status: KernelStatus,
reason: str = '', *,
db_connection: SAConnection = None,
**extra_fields,
) -> None:
data = {
'status': status,
'status_info': reason,
'status_changed': datetime.now(tzutc()),
}
data.update(extra_fields)
async with reenter_txn(self.dbpool, db_connection) as conn:
query = (
sa.update(kernels)
.values(data)
.where(
(kernels.c.id == session_id) &
(kernels.c.access_key == access_key) &
~(kernels.c.status.in_(DEAD_KERNEL_STATUSES))
)
)
await conn.execute(query)
def __init__(self, *a, **ka):
self.dict = {}
if a or ka: self.update(*a, **ka)
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def set_session_result(self, kernel_id: KernelId,
success: bool,
exit_code: int, *,
db_conn: SAConnection = None):
# TODO: store exit code?
data = {
'result': SessionResult.SUCCESS if success else SessionResult.FAILURE,
}
async with reenter_txn(self.dbpool, db_conn) as conn:
query = (
sa.update(kernels)
.values(data)
.where(kernels.c.id == kernel_id)
)
await conn.execute(query)
def __call__(self, *a, **ka):
depr('Calling ConfDict is deprecated. Use the update() method.') #0.12
self.update(*a, **ka)
return self
def rename(request: web.Request, params: Any, row: VFolderRow) -> web.Response:
dbpool = request.app['dbpool']
old_name = request.match_info['name']
access_key = request['keypair']['access_key']
domain_name = request['user']['domain_name']
user_role = request['user']['role']
user_uuid = request['user']['uuid']
new_name = params['new_name']
allowed_vfolder_types = await request.app['config_server'].get_vfolder_types()
log.info('VFOLDER.RENAME (ak:{}, vf.old:{}, vf.new:{})',
access_key, old_name, new_name)
async with dbpool.acquire() as conn:
entries = await query_accessible_vfolders(
conn, user_uuid,
user_role=user_role, domain_name=domain_name,
allowed_vfolder_types=allowed_vfolder_types)
for entry in entries:
if entry['name'] == new_name:
raise InvalidAPIParameters(
'One of your accessible vfolders already has '
'the name you requested.')
for entry in entries:
if entry['name'] == old_name:
if not entry['is_owner']:
raise InvalidAPIParameters(
'Cannot change the name of a vfolder '
'that is not owned by myself.')
query = (
vfolders.update()
.values(name=new_name)
.where(vfolders.c.id == entry['id']))
await conn.execute(query)
break
return web.Response(status=201)
def __init__(self, *a, **ka):
self._meta = {}
self._on_change = lambda name, value: None
if a or ka:
depr('Constructor does no longer accept parameters.') #0.12
self.update(*a, **ka)
def update(x, **entries):
"""Update a dict, or an object with slots, according to `entries` dict.
>>> update({'a': 1}, a=10, b=20)
{'a': 10, 'b': 20}
>>> update(Struct(a=1), a=10, b=20)
Struct(a=10, b=20)
"""
if isinstance(x, dict):
x.update(entries)
else:
x.__dict__.update(entries)
return x
#______________________________________________________________________________
# Functions on Sequences (mostly inspired by Common Lisp)
# NOTE: Sequence functions (count_if, find_if, every, some) take function
# argument first (like reduce, filter, and map).
def delete_invitation(request: web.Request, params: Any) -> web.Response:
dbpool = request.app['dbpool']
access_key = request['keypair']['access_key']
request_email = request['user']['email']
inv_id = params['inv_id']
log.info('VFOLDER.DELETE_INVITATION (ak:{}, inv:{})', access_key, inv_id)
try:
async with dbpool.acquire() as conn:
query = (sa.select([vfolder_invitations.c.inviter,
vfolder_invitations.c.invitee])
.select_from(vfolder_invitations)
.where((vfolder_invitations.c.id == inv_id) &
(vfolder_invitations.c.state == VFolderInvitationState.PENDING)))
result = await conn.execute(query)
row = await result.first()
if row is None:
raise GenericNotFound('No such vfolder invitation')
if request_email == row.inviter:
state = VFolderInvitationState.CANCELED
elif request_email == row.invitee:
state = VFolderInvitationState.REJECTED
else:
raise GenericForbidden('Cannot change other user\'s invitaiton')
query = (vfolder_invitations
.update()
.where(vfolder_invitations.c.id == inv_id)
.values(state=state))
await conn.execute(query)
except (psycopg2.IntegrityError, sa.exc.IntegrityError) as e:
raise InternalServerError(f'integrity error: {e}')
except (asyncio.CancelledError, asyncio.TimeoutError):
raise
except Exception as e:
raise InternalServerError(f'unexpected error: {e}')
return web.json_response({})
def update(self, *a, **ka):
''' If the first parameter is a string, all keys are prefixed with this
namespace. Apart from that it works just as the usual dict.update().
Example: ``update('some.namespace', key='value')`` '''
prefix = ''
if a and isinstance(a[0], basestring):
prefix = a[0].strip('.') + '.'
a = a[1:]
for key, value in dict(*a, **ka).items():
self[prefix+key] = value
def update(x, **entries):
"""Update a dict; or an object with slots; according to entries.
>>> update({'a': 1}, a=10, b=20)
{'a': 10, 'b': 20}
>>> update(Struct(a=1), a=10, b=20)
Struct(a=10, b=20)
"""
if isinstance(x, dict):
x.update(entries)
else:
x.__dict__.update(entries)
return x
#______________________________________________________________________________
# Functions on Sequences (mostly inspired by Common Lisp)
# NOTE: Sequence functions (count_if, find_if, every, some) take function
# argument first (like reduce, filter, and map).
def update_shared_vfolder(request: web.Request, params: Any) -> web.Response:
'''
Update permission for shared vfolders.
If params['perm'] is None, remove user's permission for the vfolder.
'''
dbpool = request.app['dbpool']
access_key = request['keypair']['access_key']
vfolder_id = params['vfolder']
user_uuid = params['user']
perm = params['perm']
log.info('VFOLDER.UPDATE_SHARED_VFOLDER(ak:{}, vfid:{}, uid:{}, perm:{})',
access_key, vfolder_id, user_uuid, perm)
async with dbpool.acquire() as conn:
if perm is not None:
query = (
sa.update(vfolder_permissions)
.values(permission=perm)
.where(vfolder_permissions.c.vfolder == vfolder_id)
.where(vfolder_permissions.c.user == user_uuid)
)
else:
query = (
vfolder_permissions
.delete()
.where(vfolder_permissions.c.vfolder == vfolder_id)
.where(vfolder_permissions.c.user == user_uuid)
)
await conn.execute(query)
resp = {'msg': 'shared vfolder permission updated'}
return web.json_response(resp, status=200)
def run(self, handler):
from gunicorn.app.base import Application
config = {'bind': "%s:%d" % (self.host, int(self.port))}
config.update(self.options)
class GunicornApplication(Application):
def init(self, parser, opts, args):
return config
def load(self):
return handler
GunicornApplication().run()
def begin_transaction(self, table, wait=False):
"Locks and copies table while optionally waiting for unlock."
table = self.__data.where(name=table)
assert table.first('type') is not _View, 'Views are not supported!'
lock = table.first('lock')
if wait:
lock.acquire()
with self.__lock: # Protects Critical Section
data = table.first('data')
table.update(copy=copy.deepcopy(data))
else:
with self.__lock:
assert lock.acquire(False), 'Table is locked in a transaction!'
data = table.first('data')
table.update(copy=copy.deepcopy(data))
return data
def mark_cleared(request: web.Request) -> web.Response:
dbpool = request.app['dbpool']
domain_name = request['user']['domain_name']
user_role = request['user']['role']
user_uuid = request['user']['uuid']
log_id = uuid.UUID(request.match_info['log_id'])
log.info('CLEAR')
async with dbpool.acquire() as conn, conn.begin():
query = (sa.update(error_logs)
.values(is_cleared=True))
if request['is_superadmin']:
query = query.where(error_logs.c.id == log_id)
elif user_role == UserRole.ADMIN or user_role == 'admin':
j = (groups.join(agus, groups.c.id == agus.c.group_id))
usr_query = (sa.select([agus.c.user_id])
.select_from(j)
.where([groups.c.domain_name == domain_name]))
result = await conn.execute(usr_query)
usrs = await result.fetchall()
user_ids = [g.id for g in usrs]
query = query.where((error_logs.c.user_id.in_(user_ids)) &
(error_logs.c.id == log_id))
else:
query = (query.where((error_logs.c.user_id == user_uuid) &
(error_logs.c.id == log_id)))
result = await conn.execute(query)
assert result.rowcount == 1
return web.json_response({'success': True}, status=200)
def prepare(self, **options):
""" Run preparations (parsing, caching, ...).
It should be possible to call this again to refresh a template or to
update settings.
"""
raise NotImplementedError
def test():
"Runs several groups of tests of the database engine."
# Test simple statements in SQL.
persons = test_basic_sql()
# Test various ways to select rows.
test_row_selection(persons)
# Test the four different types of joins in SQL.
orders = test_all_joins(persons)
# Test unstructured ways of joining tables together.
test_table_addition(persons, orders)
# Test creation and manipulation of databases.
test_database_support()
# Load and run some test on the sample Northwind database.
northwind = test_northwind()
# Test different date operations that can be performed.
test_date_functionality()
# Test various functions that operate on specified column.
test_column_functions()
if northwind:
# Test ability to select columns with function processing.
test_generic_column_functions(persons, northwind)
# Test Database2 instances that support transactions.
nw2 = test_transactional_database()
# Allow for interaction at the end of the test.
globals().update(locals())
def filehash(filename, block_size=65536, algorithm='sha256'):
"""
Computes the hash value for a file by using a specified hash algorithm.
:param filename: filename
:type filename: str
:param block_size: blocksize used to compute file hash
:type block_size: int
:param algorithm: hash algorithms like 'sha256' or 'md5' etc.
:type algorithm: str
:return: None
:History: 2019-Mar-12 - Written - Henry Leung (University of Toronto)
"""
algorithm = algorithm.lower()
if algorithm not in hashlib.algorithms_guaranteed:
raise ValueError(f"{algorithm} is an unsupported hashing algorithm")
func_algorithm = getattr(hashlib, algorithm)()
with open(filename, 'rb') as f:
for block in iter(lambda: f.read(block_size), b''):
func_algorithm.update(block)
return func_algorithm.hexdigest()
def prepare(self, **options):
from mako.template import Template
from mako.lookup import TemplateLookup
options.update({'input_encoding':self.encoding})
options.setdefault('format_exceptions', bool(DEBUG))
lookup = TemplateLookup(directories=self.lookup, **options)
if self.source:
self.tpl = Template(self.source, lookup=lookup, **options)
else:
self.tpl = Template(uri=self.name, filename=self.filename, lookup=lookup, **options)
def test_date_functionality():
"Tests different date operations that can be performed."
# Create an orderz table to test the date type.
orderz = Table(('OrderId', int), ('ProductName', str), ('OrderDate', date))
orderz.insert(1, 'Geitost', date(2008, 11, 11))
orderz.insert(2, 'Camembert Pierrot', date(2008, 11, 9))
orderz.insert(3, 'Mozzarella di Giovanni', date(2008, 11, 11))
orderz.insert(4, 'Mascarpone Fabioloi', date(2008, 10, 29))
# Query the table for a specific date.
orderz.where(ROW.OrderDate == date(2008, 11, 11)).print()
# Update the orderz table so that times are present with the dates.
orderz.alter_column('OrderDate', datetime)
orderz.where(ROW.OrderId == 1) \
.update(OrderDate=datetime(2008, 11, 11, 13, 23, 44))
orderz.where(ROW.OrderId == 2) \
.update(OrderDate=datetime(2008, 11, 9, 15, 45, 21))
orderz.where(ROW.OrderId == 3) \
.update(OrderDate=datetime(2008, 11, 11, 11, 12, 1))
orderz.where(ROW.OrderId == 4) \
.update(OrderDate=datetime(2008, 10, 29, 14, 56, 59))
# Query the table with a datetime object this time.
orderz.where(ROW.OrderDate == datetime(2008, 11, 11)).print()
def md5sum(downfile):
'''
文件的 md5 哈希值
:param downfile:
:return:
'''
import hashlib
md5_l = hashlib.md5()
with open(downfile, mode="rb") as fp:
by = fp.read()
md5_l.update(by)
ret = md5_l.hexdigest()
return ret
def render(self, *args, **kwargs):
for dictarg in args: kwargs.update(dictarg)
_defaults = self.defaults.copy()
_defaults.update(kwargs)
return self.tpl.render(**_defaults)
def __copy__(self):
copy = defaultdict(self.default)
copy.update(self)
return copy
def update_to(self, b=1, bsize=1, tsize=None):
"""
b : int, optional
Number of blocks transferred so far [default: 1].
bsize : int, optional
Size of each block (in tqdm units) [default: 1].
tsize : int, optional
Total size (in tqdm units). If [default: None] remains unchanged.
"""
if tsize is not None:
self.total = tsize
self.update(b * bsize - self.n) # will also set self.n = b * bsize
def prepare(self, filters=None, tests=None, globals={}, **kwargs):
from jinja2 import Environment, FunctionLoader
if 'prefix' in kwargs: # TODO: to be removed after a while
raise RuntimeError('The keyword argument `prefix` has been removed. '
'Use the full jinja2 environment name line_statement_prefix instead.')
self.env = Environment(loader=FunctionLoader(self.loader), **kwargs)
if filters: self.env.filters.update(filters)
if tests: self.env.tests.update(tests)
if globals: self.env.globals.update(globals)
if self.source:
self.tpl = self.env.from_string(self.source)
else:
self.tpl = self.env.get_template(self.filename)
def __copy__(self):
copy = DefaultDict(self.default)
copy.update(self)
return copy
def update_to(self, downloaded=0, total_size=None):
"""
b : int, optional
Number of blocks transferred so far [default: 1].
bsize : int, optional
Size of each block (in tqdm units) [default: 1].
tsize : int, optional
Total size (in tqdm units). If [default: None] remains unchanged.
"""
if total_size is not None:
self.total = total_size
# self.ascii = True
self.update(downloaded - self.n) # will also set self.n = b * bsize
def render(self, *args, **kwargs):
for dictarg in args: kwargs.update(dictarg)
_defaults = self.defaults.copy()
_defaults.update(kwargs)
return self.tpl.render(**_defaults)
def update_to(self, b=1, bsize=1, tsize=None):
"""
b : int, optional
Number of blocks transferred so far [default: 1].
bsize : int, optional
Size of each block (in tqdm units) [default: 1].
tsize : int, optional
Total size (in tqdm units). If [default: None] remains unchanged.
"""
if tsize is not None:
self.total = tsize
self.update(b * bsize - self.n) # will also set self.n = b * bsize
def _include(self, _env, _name=None, **kwargs):
if _name is None:
depr('Rebase function called without arguments.'
' You were probably looking for {{base}}?', True) #0.12
env = _env.copy()
env.update(kwargs)
if _name not in self.cache:
self.cache[_name] = self.__class__(name=_name, lookup=self.lookup)
return self.cache[_name].execute(env['_stdout'], env)
def __copy__(self):
copy = DefaultDict(self.default)
copy.update(self)
return copy
def update_to(self, b=1, bsize=1, tsize=None):
"""
b : int, optional
Number of blocks transferred so far [default: 1].
bsize : int, optional
Size of each block (in tqdm units) [default: 1].
tsize : int, optional
Total size (in tqdm units). If [default: None] remains unchanged.
"""
if tsize is not None:
self.total = tsize
self.update(b * bsize - self.n) # will also set self.n = b * bsize
def execute(self, _stdout, kwargs):
env = self.defaults.copy()
env.update(kwargs)
env.update({'_stdout': _stdout, '_printlist': _stdout.extend,
'include': functools.partial(self._include, env),
'rebase': functools.partial(self._rebase, env), '_rebase': None,
'_str': self._str, '_escape': self._escape, 'get': env.get,
'setdefault': env.setdefault, 'defined': env.__contains__ })
eval(self.co, env)
if env.get('_rebase'):
subtpl, rargs = env.pop('_rebase')
rargs['base'] = ''.join(_stdout) #copy stdout
del _stdout[:] # clear stdout
return self._include(env, subtpl, **rargs)
return env
def _hash_file(fpath, algorithm="sha256", chunk_size=65535):
"""Calculates a file sha256 or md5 hash.
# Example
```python
>>> from keras.data_utils import _hash_file
>>> _hash_file("/path/to/file.zip")
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
```
# Arguments
fpath: path to the file being validated
algorithm: hash algorithm, one of "auto", "sha256", or "md5".
The default "auto" detects the hash algorithm in use.
chunk_size: Bytes to read at a time, important for large files.
# Returns
The file hash
"""
hasher = hashlib.md5()
with open(fpath, "rb") as fpath_file:
for chunk in iter(lambda: fpath_file.read(chunk_size), b""):
hasher.update(chunk)
return hasher.hexdigest()
def render(self, *args, **kwargs):
""" Render the template using keyword arguments as local variables. """
env = {}; stdout = []
for dictarg in args: env.update(dictarg)
env.update(kwargs)
self.execute(stdout, env)
return ''.join(stdout)
def __init__(self, order=min, f=lambda x: x):
update(self, A=[], order=order, f=f)
def my_hook(t):
"""Wraps tqdm instance.
Don't forget to close() or __exit__()
the tqdm instance once you're done with it (easiest using `with` syntax).
Example
-------
>>> with tqdm(...) as t:
... reporthook = my_hook(t)
... urllib.urlretrieve(..., reporthook=reporthook)
"""
last_b = [0]
def update_to(b=1, bsize=1, tsize=None):
"""
b : int, optional
Number of blocks transferred so far [default: 1].
bsize : int, optional
Size of each block (in tqdm units) [default: 1].
tsize : int, optional
Total size (in tqdm units). If [default: None] remains unchanged.
"""
if tsize is not None:
t.total = tsize
t.update((b - last_b[0]) * bsize)
last_b[0] = b
return update_to
def __init__(self, body='', status=None, header=None, **headers):
if header or 'output' in headers:
depr('Call signature changed (for the better)')
if header: headers.update(header)
if 'output' in headers: body = headers.pop('output')
super(HTTPResponse, self).__init__(body, status, **headers)
def update(self, other):
for e in other:
self.add(e)
return self
def update_to(self, b=1, bsize=1, tsize=None):
"""
b : int, optional
Number of blocks transferred so far [default: 1].
bsize : int, optional
Size of each block (in tqdm units) [default: 1].
tsize : int, optional
Total size (in tqdm units). If [default: None] remains unchanged.
"""
if tsize is not None:
self.total = tsize
self.update(b * bsize - self.n) # will also set self.n = b * bsize
def __init__(self, name, impmask):
''' Create a virtual package that redirects imports (see PEP 302). '''
self.name = name
self.impmask = impmask
self.module = sys.modules.setdefault(name, imp.new_module(name))
self.module.__dict__.update({'__file__': __file__, '__path__': [],
'__all__': [], '__loader__': self})
sys.meta_path.append(self)
def __copy__(self):
copy = DefaultDict(self.default)
copy.update(self)
return copy
def update_to(self, b=1, bsize=1, tsize=None):
"""
b : int, optional
Number of blocks transferred so far [default: 1].
bsize : int, optional
Size of each block (in tqdm units) [default: 1].
tsize : int, optional
Total size (in tqdm units). If [default: None] remains unchanged.
"""
if tsize is not None:
self.total = tsize
self.update(b * bsize - self.n) # will also set self.n = b * bsize
def __init__(self, *a, **ka):
self.dict = {}
if a or ka: self.update(*a, **ka)
def __init__(self, **entries):
self.__dict__.update(entries)
def update_to(self, b=1, bsize=1, tsize=None):
"""
b : int, optional
Number of blocks transferred so far [default: 1].
bsize : int, optional
Size of each block (in tqdm units) [default: 1].
tsize : int, optional
Total size (in tqdm units). If [default: None] remains unchanged.
"""
if tsize is not None:
self.total = tsize
self.update(b * bsize - self.n) # will also set self.n = b * bsize
def __init__(self, order=min, f=lambda x: x):
update(self, A=[], order=order, f=f)
def update_to(self, b=1, bsize=1, tsize=None):
"""Reports update statistics on the download progress.
Args:
b (int): Number of blocks transferred so far [default: 1].
bsize (int): Size of each block (in tqdm units) [default: 1].
tsize (int): Total size (in tqdm units). If [default: None] remains unchanged.
"""
if tsize is not None:
self.total = tsize
self.update(b * bsize - self.n) # will also set self.n = b * bsize
def __extend_data(self):
"Adds columns to internal table as necessary."
if ('type', type) not in self.__data.schema:
self.__data.alter_add('type', type)
for name, data in rows(self.__data('name', 'data')):
self.__data.where(name=name).update(type=type(data))
self.__data.alter_add('lock', _Lock)
self.__data.alter_add('copy', object)
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def __commit(table):
"Deletes the reserve copy of a table."
table.update(copy=object())
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def __rollback(table):
"Restores table from copy and deletes the copy."
table.update(data=table.first('copy'), copy=object())
########################################################################
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def update(self, **assignments):
"Changes all present rows with given assignments."
assign = []
for name, value in assignments.items():
data_type, index = self.__columns[name]
assert isinstance(value, data_type), \
'Wrong datatype: {} ({!r}, {!r})'.format(name, value, data_type)
assign.append((index, value))
for row in self.__data_area.values():
for index, value in assign:
row[index] = value
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def test_basic_sql():
"Tests simple statements in SQL."
# Test create table statement.
persons = Table(('P_Id', int), ('LastName', str), ('FirstName', str),
('Address', str), ('City', str))
# Populate the table with rows.
persons.insert(1, 'Hansen', 'Ola', 'Timoteivn 10', 'Sandnes')
persons.insert(2, 'Svendson', 'Tove', 'Borgvn 23', 'Sandnes')
persons.insert(3, 'Pettersen', 'Kari', 'Storgt 20', 'Stavanger')
persons.print()
# Test the select statement.
persons.select('LastName', 'FirstName').print()
persons.select().print()
# Test the distinct statement.
persons.select('City').distinct().print()
# Test the where clause.
persons.where(ROW.City == 'Sandnes').print()
# Test the and operator.
persons.where((ROW.FirstName == 'Tove') &
(ROW.LastName == 'Svendson')).print()
# Test the or operator.
persons.where((ROW.FirstName == 'Tove') | (ROW.FirstName == 'Ola')).print()
# Test both and & or operators.
persons.where((ROW.LastName == 'Svendson') &
((ROW.FirstName == 'Tove') |
(ROW.FirstName == 'Ola'))).print()
# Test order by statement.
persons.insert(4, 'Nilsen', 'Tom', 'Vingvn 23', 'Stavanger')
persons.order_by('LastName').table().print()
persons.order_by('LastName', True).table().print()
# Test insert statement.
persons.insert(5, 'Nilsen', 'Johan', 'Bakken 2', 'Stavanger')
persons.print()
persons.insert(P_Id=6, LastName='Tjessem', FirstName='Jakob')
persons.print()
# Test update statement.
persons.where((ROW.LastName == 'Tjessem') &
(ROW.FirstName == 'Jakob')).update(Address='Nissestien 67',
City='Sandnes')
persons.print()
copy = persons.order_by('P_Id').table()
copy.update(Address='Nissestien 67', City='Sandnes')
copy.print()
# Test delete statement.
copy = persons.order_by('P_Id').table()
copy.delete((ROW.LastName == 'Tjessem') &
(ROW.FirstName == 'Jakob')).print()
copy.truncate().print()
return persons
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def test_northwind():
"Loads and runs some test on the sample Northwind database."
import os, imp
# Patch the module namespace to recognize this file.
name = os.path.splitext(os.path.basename(sys.argv[0]))[0]
module = imp.new_module(name)
vars(module).update(globals())
sys.modules[name] = module
# Load a Northwind database for various testing purposes.
try:
northwind = Database.load('northwind.db')
except IOError:
return
# Create and test a current product list view.
northwind.create('Current Product List', lambda db: db.Products.where(
ROW.Discontinued.NOT).select('ProductID', 'ProductName'))
northwind['Current Product List'].print()
# Find all products having an above-average price.
def above_average_price(db):
return db.Products.where(ROW.UnitPrice > db.Products.avg('UnitPrice')) \
.select('ProductName', 'UnitPrice')
northwind.create('Products Above Average Price', above_average_price)
northwind['Products Above Average Price'].print()
# Calculate total sale per category in 1997.
def category_sales_for_1997(db):
result = Table(('CategoryName', str),
('CategorySales', decimal.Decimal))
for table in db['Product Sales For 1997'] \
.group_by('Categories.CategoryName'):
name = next(rows(table.select('Categories.CategoryName')))[0]
total = table.sum_('ProductSales')
result.insert(name, total)
return result
northwind.create('Category Sales For 1997', category_sales_for_1997)
northwind['Category Sales For 1997'].print()
# Show just the Beverages Category from the previous view.
northwind['Category Sales For 1997'].where(
ROW.CategoryName == 'Beverages').print()
# Add the Category column to the Current Product List view.
northwind.create_or_replace('Current Product List', lambda db: \
db['Products View'].where(ROW.Discontinued.NOT) \
.select('ProductID', 'ProductName', 'Category'))
northwind['Current Product List'].print()
# Drop the Category Sales For 1997 view.
northwind.drop('Category Sales For 1997')
return northwind
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def visualize_2D_trip(self, trip):
plt.figure(figsize=(30,30))
rcParams.update({'font.size': 22})
# Plot cities
plt.scatter(trip[:,0], trip[:,1], s=200)
# Plot tour
tour=np.array(list(range(len(trip))) + [0])
X = trip[tour, 0]
Y = trip[tour, 1]
plt.plot(X, Y,"--", markersize=100)
# Annotate cities with order
labels = range(len(trip))
for i, (x, y) in zip(labels,(zip(X,Y))):
plt.annotate(i,xy=(x, y))
plt.xlim(0,100)
plt.ylim(0,100)
plt.show()
# Heatmap of permutations (x=cities; y=steps)
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
self.updates = [K.update_add(self.iterations, 1)]
t = K.cast(self.iterations, K.floatx()) + 1
lr_t = self.learning_rate * (K.sqrt(1. - K.pow(self.beta_2, t)) / (1. - K.pow(self.beta_1, t)))
ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
self.weights = [self.iterations] + ms + vs
for p, g, m, v in zip(params, grads, ms, vs):
m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)
p_t = lr_t * m_t / (K.sqrt(v_t) + self.epsilon)
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
self.updates.append(K.update_sub(p, p_t))
return self.updates
def visualize_2D_trip(self,trip,tw_open,tw_close):
plt.figure(figsize=(30,30))
rcParams.update({'font.size': 22})
# Plot cities
colors = ['red'] # Depot is first city
for i in range(len(tw_open)-1):
colors.append('blue')
plt.scatter(trip[:,0], trip[:,1], color=colors, s=200)
# Plot tour
tour=np.array(list(range(len(trip))) + [0])
X = trip[tour, 0]
Y = trip[tour, 1]
plt.plot(X, Y,"--", markersize=100)
# Annotate cities with TW
tw_open = np.rint(tw_open)
tw_close = np.rint(tw_close)
time_window = np.concatenate((tw_open,tw_close),axis=1)
for tw, (x, y) in zip(time_window,(zip(X,Y))):
plt.annotate(tw,xy=(x, y))
plt.xlim(0,60)
plt.ylim(0,60)
plt.show()
# Heatmap of permutations (x=cities; y=steps)
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def merge_updates(updates):
"""Average repeated updates of the same variable"""
merged_updates = {}
for update in updates:
variable, value = unpack_assignment(update)
key = variable_key(variable)
if key not in merged_updates:
merged_updates[key] = [variable, []]
merged_updates[key][1].append(value)
ret = []
for k, v in iteritems(merged_updates):
variable = v[0]
values = v[1]
n = len(values)
if n == 1:
ret.append(K.update(variable, value[0]))
else:
ret.append(K.update(variable, sum(values) / n))
return ret
def visualize_sampling(self, permutations):
max_length = len(permutations[0])
grid = np.zeros([max_length,max_length]) # initialize heatmap grid to 0
transposed_permutations = np.transpose(permutations)
for t, cities_t in enumerate(transposed_permutations): # step t, cities chosen at step t
city_indices, counts = np.unique(cities_t,return_counts=True,axis=0)
for u,v in zip(city_indices, counts):
grid[t][u]+=v # update grid with counts from the batch of permutations
# plot heatmap
fig = plt.figure()
rcParams.update({'font.size': 22})
ax = fig.add_subplot(1,1,1)
ax.set_aspect('equal')
plt.imshow(grid, interpolation='nearest', cmap='gray')
plt.colorbar()
plt.title('Sampled permutations')
plt.ylabel('Time t')
plt.xlabel('City i')
plt.show()
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def call(self, inputs, training=None):
z, gamma_k = inputs
gamma_k_sum = K.sum(gamma_k)
est_phi = K.mean(gamma_k, axis=0)
est_mu = K.dot(K.transpose(gamma_k), z) / gamma_k_sum
est_sigma = K.dot(K.transpose(z - est_mu),
gamma_k * (z - est_mu)) / gamma_k_sum
est_sigma = est_sigma + (K.random_normal(shape=(K.int_shape(z)[1], 1), mean=1e-3, stddev=1e-4) * K.eye(K.int_shape(z)[1]))
self.add_update(K.update(self.phi, est_phi), inputs)
self.add_update(K.update(self.mu, est_mu), inputs)
self.add_update(K.update(self.sigma, est_sigma), inputs)
est_sigma_diag_inv = K.eye(K.int_shape(self.sigma)[0]) / est_sigma
self.add_loss(self.lambd_diag * K.sum(est_sigma_diag_inv), inputs)
phi = K.in_train_phase(est_phi, self.phi, training)
mu = K.in_train_phase(est_mu, self.mu, training)
sigma = K.in_train_phase(est_sigma, self.sigma, training)
return GaussianMixtureComponent._calc_component_density(z, phi, mu, sigma)
def plot_class_ROC(fpr, tpr, roc_auc, class_idx, labels):
from matplotlib import rcParams
# Make room for xlabel which is otherwise cut off
rcParams.update({'figure.autolayout': True})
plt.figure()
lw = 2
plt.plot(fpr[class_idx], tpr[class_idx], color='darkorange',
lw=lw, label='ROC curve (area = %0.2f)' % roc_auc[class_idx])
plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Receiver operating characteristic of class {}'.format(labels[class_idx]))
plt.legend(loc="lower right")
plt.tight_layout()
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def apply_box_deltas_graph(boxes, deltas):
"""Applies the given deltas to the given boxes.
boxes: [N, (y1, x1, y2, x2)] boxes to update
deltas: [N, (dy, dx, log(dh), log(dw))] refinements to apply
"""
# Convert to y, x, h, w
height = boxes[:, 2] - boxes[:, 0]
width = boxes[:, 3] - boxes[:, 1]
center_y = boxes[:, 0] + 0.5 * height
center_x = boxes[:, 1] + 0.5 * width
# Apply deltas
center_y += deltas[:, 0] * height
center_x += deltas[:, 1] * width
height *= tf.exp(deltas[:, 2])
width *= tf.exp(deltas[:, 3])
# Convert back to y1, x1, y2, x2
y1 = center_y - 0.5 * height
x1 = center_x - 0.5 * width
y2 = y1 + height
x2 = x1 + width
result = tf.stack([y1, x1, y2, x2], axis=1, name="apply_box_deltas_out")
return result
def add_theme(self, other, inplace=False):
"""Add themes together.
Subclasses should not override this method.
This will be called when adding two instances of class 'theme'
together.
A complete theme will annihilate any previous themes. Partial themes
can be added together and can be added to a complete theme.
"""
if other.complete:
return other
theme_copy = self if inplace else deepcopy(self)
theme_copy.themeables.update(deepcopy(other.themeables))
return theme_copy
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def apply_box_deltas_graph(boxes, deltas):
"""Applies the given deltas to the given boxes.
boxes: [N, (y1, x1, y2, x2)] boxes to update
deltas: [N, (dy, dx, log(dh), log(dw))] refinements to apply
"""
# Convert to y, x, h, w
height = boxes[:, 2] - boxes[:, 0]
width = boxes[:, 3] - boxes[:, 1]
center_y = boxes[:, 0] + 0.5 * height
center_x = boxes[:, 1] + 0.5 * width
# Apply deltas
center_y += deltas[:, 0] * height
center_x += deltas[:, 1] * width
height *= tf.exp(deltas[:, 2])
width *= tf.exp(deltas[:, 3])
# Convert back to y1, x1, y2, x2
y1 = center_y - 0.5 * height
x1 = center_x - 0.5 * width
y2 = y1 + height
x2 = x1 + width
result = tf.stack([y1, x1, y2, x2], axis=1, name="apply_box_deltas_out")
return result
def set_pub_style():
"""formatting helper function that can be used to save publishable figures"""
set_figure_params('dynamo', background='white')
matplotlib.use('cairo')
matplotlib.rcParams.update({'font.size': 4})
params = {'legend.fontsize': 4,
'legend.handlelength': 0.5}
matplotlib.rcParams.update(params)
params = {'axes.labelsize': 6,
'axes.titlesize':6,
'xtick.labelsize':6,
'ytick.labelsize':6,
'axes.titlepad': 1,
'axes.labelpad': 1
}
matplotlib.rcParams.update(params)
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def add_weightnorm_param_updates(updates, new_V_param, new_g_param, W, V_scaler):
ps = K.get_variable_shape(new_V_param)
norm_axes = [i for i in range(len(ps) - 1)]
# update W and V_scaler
new_V_norm = tf.sqrt(tf.reduce_sum(tf.square(new_V_param), norm_axes))
new_V_scaler = new_g_param / new_V_norm
new_W = tf.reshape(new_V_scaler, [1] * len(norm_axes) + [-1]) * new_V_param
updates.append(K.update(W, new_W))
updates.append(K.update(V_scaler, new_V_scaler))
# data based initialization for a given Keras model
def visualize_sampling(self,permutations):
max_length = len(permutations[0])
grid = np.zeros([max_length,max_length]) # initialize heatmap grid to 0
transposed_permutations = np.transpose(permutations)
for t, cities_t in enumerate(transposed_permutations): # step t, cities chosen at step t
city_indices, counts = np.unique(cities_t,return_counts=True,axis=0)
for u,v in zip(city_indices, counts):
grid[t][u]+=v # update grid with counts from the batch of permutations
# plot heatmap
fig = plt.figure()
rcParams.update({'font.size': 22})
ax = fig.add_subplot(1,1,1)
ax.set_aspect('equal')
plt.imshow(grid, interpolation='nearest', cmap='gray')
plt.colorbar()
plt.title('Sampled permutations')
plt.ylabel('Time t')
plt.xlabel('City i')
plt.show()
# Heatmap of attention (x=cities; y=steps)
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def data_based_init(model, input):
# input can be dict, numpy array, or list of numpy arrays
if type(input) is dict:
feed_dict = input
elif type(input) is list:
feed_dict = {tf_inp: np_inp for tf_inp,np_inp in zip(model.inputs,input)}
else:
feed_dict = {model.inputs[0]: input}
# add learning phase if required
if model.uses_learning_phase and K.learning_phase() not in feed_dict:
feed_dict.update({K.learning_phase(): 1})
# get all layer name, output, weight, bias tuples
layer_output_weight_bias = []
for l in model.layers:
if hasattr(l, 'W') and hasattr(l, 'b'):
assert(l.built)
layer_output_weight_bias.append( (l.name,l.get_output_at(0),l.W,l.b) ) # if more than one node, only use the first
# iterate over our list and do data dependent init
sess = K.get_session()
for l,o,W,b in layer_output_weight_bias:
print('Performing data dependent initialization for layer ' + l)
m,v = tf.nn.moments(o, [i for i in range(len(o.get_shape())-1)])
s = tf.sqrt(v + 1e-10)
updates = tf.group(W.assign(W/tf.reshape(s,[1]*(len(W.get_shape())-1)+[-1])), b.assign((b-m)/s))
sess.run(updates, feed_dict)