UPDATE: Turned my solution into a stand-alone python script.
This solution has saved me more than once. Hopefully others find it useful. This python script will find any jupyter kernel using more than cpu_threshold
CPU and prompts the user to send a SIGINT
to the kernel (KeyboardInterrupt). It will keep sending SIGINT
until the kernel's cpu usage goes below cpu_threshold
. If there are multiple misbehaving kernels it will prompt the user to interrupt each of them (ordered by highest CPU usage to lowest). A big thanks goes to gcbeltramini for writing code to find the name of a jupyter kernel using the jupyter api. This script was tested on MACOS with python3 and requires jupyter notebook, requests, json and psutil.
Put the script in your home directory and then usage looks like:
python ~/interrupt_bad_kernels.py
Interrupt kernel chews cpu.ipynb; PID: 57588; CPU: 2.3%? (y/n) y
Script code below:
from os import getpid, kill
from time import sleep
import re
import signal
from notebook.notebookapp import list_running_servers
from requests import get
from requests.compat import urljoin
import ipykernel
import json
import psutil
def get_active_kernels(cpu_threshold):
"""Get a list of active jupyter kernels."""
active_kernels = []
pids = psutil.pids()
my_pid = getpid()
for pid in pids:
if pid == my_pid:
continue
try:
p = psutil.Process(pid)
cmd = p.cmdline()
for arg in cmd:
if arg.count('ipykernel'):
cpu = p.cpu_percent(interval=0.1)
if cpu > cpu_threshold:
active_kernels.append((cpu, pid, cmd))
except psutil.AccessDenied:
continue
return active_kernels
def interrupt_bad_notebooks(cpu_threshold=0.2):
"""Interrupt active jupyter kernels. Prompts the user for each kernel."""
active_kernels = sorted(get_active_kernels(cpu_threshold), reverse=True)
servers = list_running_servers()
for ss in servers:
response = get(urljoin(ss['url'].replace('localhost', '127.0.0.1'), 'api/sessions'),
params={'token': ss.get('token', '')})
for nn in json.loads(response.text):
for kernel in active_kernels:
for arg in kernel[-1]:
if arg.count(nn['kernel']['id']):
pid = kernel[1]
cpu = kernel[0]
interrupt = input(
'Interrupt kernel {}; PID: {}; CPU: {}%? (y/n) '.format(nn['notebook']['path'], pid, cpu))
if interrupt.lower() == 'y':
p = psutil.Process(pid)
while p.cpu_percent(interval=0.1) > cpu_threshold:
kill(pid, signal.SIGINT)
sleep(0.5)
if __name__ == '__main__':
interrupt_bad_notebooks()