97 lines
3.1 KiB
Python
97 lines
3.1 KiB
Python
|
import subprocess, time, os, threading, math
|
||
|
|
|
||
|
|
from intra.system import system_manager
|
||
|
|
from intra.cgroup import cgroup_manager
|
||
|
|
from intra.billing import billing_manager
|
||
|
|
|
||
|
|
class smart_controller:
|
||
|
|
|
||
|
|
def set_policy(policy):
|
||
|
|
smart_controller.policy = policy
|
||
|
|
|
||
|
|
def start(interval = 4):
|
||
|
|
thread = threading.Thread(target = smart_controller.smart_control_forever, args = [interval])
|
||
|
|
thread.setDaemon(True)
|
||
|
|
thread.start()
|
||
|
|
return thread
|
||
|
|
|
||
|
|
def smart_control_forever(interval):
|
||
|
|
last_live = []
|
||
|
|
while True:
|
||
|
|
time.sleep(interval)
|
||
|
|
try:
|
||
|
|
mem_usage_mapping = {}
|
||
|
|
live = cgroup_manager.get_cgroup_containers()
|
||
|
|
for item in live:
|
||
|
|
try:
|
||
|
|
last_live.remove(item)
|
||
|
|
except:
|
||
|
|
pass
|
||
|
|
try:
|
||
|
|
cgroup_manager.protect_container_oom(item)
|
||
|
|
sample = cgroup_manager.get_container_sample(item)
|
||
|
|
mem_usage_mapping[item] = math.ceil(sample['mem_page_sample'] * 1e-6)
|
||
|
|
billing_manager.add_usage_sample(item, sample, interval)
|
||
|
|
except:
|
||
|
|
pass
|
||
|
|
for item in last_live:
|
||
|
|
billing_manager.clean_dead_node(item)
|
||
|
|
last_live = live
|
||
|
|
is_ready = True
|
||
|
|
|
||
|
|
memory_available = system_manager.get_available_memsw()
|
||
|
|
if memory_available['Mbytes'] <= 0:
|
||
|
|
size_in_gb = int(math.ceil(-memory_available['Mbytes'] / 1024 / 16) * 16)
|
||
|
|
print("[warning]", 'overloaded containers, auto-extending %d G memsw.' % size_in_gb)
|
||
|
|
system_manager.extend_swap(size_in_gb)
|
||
|
|
|
||
|
|
total_score = 0.0
|
||
|
|
score_mapping = {}
|
||
|
|
for item in live:
|
||
|
|
score = max(1e-8, smart_controller.policy.get_score_by_uuid(item))
|
||
|
|
score_mapping[item] = score
|
||
|
|
print(item, "(score/cpu)", score)
|
||
|
|
total_score += score
|
||
|
|
|
||
|
|
# CPU Scoring
|
||
|
|
for item in live:
|
||
|
|
ceof = score_mapping[item] / total_score
|
||
|
|
cgroup_manager.set_container_cpu_priority_limit(item, ceof)
|
||
|
|
|
||
|
|
# Iterative Memory Scoring
|
||
|
|
free_mem = system_manager.get_total_physical_memory_for_containers()['Mbytes']
|
||
|
|
local_nodes = live
|
||
|
|
mem_alloc = {}
|
||
|
|
for item in live:
|
||
|
|
mem_alloc[item] = 0
|
||
|
|
|
||
|
|
while free_mem > 0 and len(local_nodes) > 0:
|
||
|
|
excess_mem = 0
|
||
|
|
next_local_nodes = []
|
||
|
|
for item in local_nodes:
|
||
|
|
mem_alloc[item] += int(math.floor(free_mem * score_mapping[item] / total_score))
|
||
|
|
if mem_alloc[item] >= mem_usage_mapping[item]:
|
||
|
|
excess_mem += mem_alloc[item] - mem_usage_mapping[item]
|
||
|
|
mem_alloc[item] = mem_usage_mapping[item]
|
||
|
|
else:
|
||
|
|
next_local_nodes.append(item)
|
||
|
|
free_mem = excess_mem
|
||
|
|
local_nodes = next_local_nodes
|
||
|
|
|
||
|
|
for item in live:
|
||
|
|
mem_alloc[item] += int(math.floor(free_mem * score_mapping[item] / total_score))
|
||
|
|
cgroup_manager.set_container_physical_memory_limit(item, mem_alloc[item])
|
||
|
|
print(item, "(malloc:usage)", mem_alloc[item], mem_usage_mapping[item])
|
||
|
|
|
||
|
|
if len(live) > 0:
|
||
|
|
print("-------------------------------")
|
||
|
|
|
||
|
|
except:
|
||
|
|
pass
|
||
|
|
|
||
|
|
|
||
|
|
# echo "8:0 1000" > /sys/fs/cgroup/blkio/lxc/docklet-1-0/blkio.throttle.write_bps_device
|
||
|
|
# https://www.kernel.org/doc/Documentation/devices.txt
|
||
|
|
# while true; do clear; cat /sys/fs/cgroup/blkio/lxc/docklet-1-0/blkio.throttle.io_service_bytes; sleep 0.5; done
|
||
|
|
# hugetlb, net_cls, net_prio, /sbin/tc
|