ampere_research/pytorch/run.py

import data_stat
from data_stat import Stat, Cpu

import argparse
import os, sys
import subprocess, signal
import json
import time

parser = argparse.ArgumentParser()
parser.add_argument('cpu', choices=[x.name.lower() for x in Cpu])
parser.add_argument('matrix_file')
parser.add_argument('base_iterations', type=int)
parser.add_argument('min_time_s', type=int)
parser.add_argument('baseline_time_s', type=int)
parser.add_argument('baseline_delay_s', type=int)
#parser.add_argument('--perf', action='store_true')
parser.add_argument('--power', action='store_true')
parser.add_argument('-d', '--debug', action='store_true')
args = parser.parse_args()
args.cpu = Cpu[args.cpu.upper()]

python = {
        Cpu.ALTRA: 'python3',
        Cpu.EPYC_7313P: 'python3.11',
        Cpu.XEON_4216: 'python3.11'
}
perf = ['perf', 'stat']
perf_args = {
        Cpu.ALTRA: [
            ['-d', '-d'],
            ['-M', 'branch_misprediction_ratio'],
            ['-M', 'dtlb_walk_ratio,itlb_walk_ratio'],
            ['-M', 'l1d_cache_miss_ratio,l1i_cache_miss_ratio'],
            ['-M', 'l2_cache_miss_ratio,l2_tlb_miss_ratio,ll_cache_read_miss_ratio']]
}

def program(cpu: Cpu, matrix_file: str, iterations: int) -> list:
    if cpu == Cpu.ALTRA:
        return [
            'apptainer', 'run', 'pytorch-altra.sif', '-c',
            'numactl --cpunodebind=0 --membind=0 '
            + f'python3 spmv.py {matrix_file} {iterations}']
    elif cpu == Cpu.EPYC_7313P: 
        return [
            'apptainer', 'run', 'pytorch-epyc_7313p.sif',
            'python3', 'spmv.py', f'{matrix_file}', f'{iterations}']
    elif cpu == Cpu.XEON_4216:
        return [
            'apptainer', 'run', 'pytorch-xeon_4216.sif',
            'numactl', '--cpunodebind=0', '--membind=0',
            'python3', 'spmv.py', f'{matrix_file}', f'{iterations}']

def baseline_power(cpu: Cpu, baseline_time_s: int) -> list:
    power_process = subprocess.Popen(['./power.sh', str(baseline_time_s)],
            stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True)
    return [float(x) for x in power_process.communicate()[0].strip().split('\n') if len(x) != 0]

def run_program(program: list[str]) -> tuple[dict, str]:
    if args.debug:
        print(program)
    process = subprocess.run(program,
            stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
    process.check_returncode()

    if args.debug:
        print(process.stdout)
        print(process.stderr)
    return (json.loads(process.stdout), process.stderr)

def trapezoidal_rule(power: list[float], time_s: float) -> float:
    result = 0.0
    relevant_power = power[-int(time_s):]

    assert(time_s >= 2)
    assert(len(relevant_power) >= 2)
    assert(len(power) >= time_s)

    for pair in zip(relevant_power, relevant_power[1:]):
        result += 0.5 * (pair[0] + pair[1])
    result += (time_s % 1) * (power[-1])
    return result

result = dict()
result[Stat.CPU.name] = args.cpu.name

iterations = args.base_iterations
program_result = run_program(program(args.cpu, args.matrix_file, iterations))
while program_result[0][Stat.TIME_S.name] < args.min_time_s:
    # Increase the number of iterations by difference between the current time taken and the desired time.
    iterations *= 1 / ((args.min_time_s - program_result[0][Stat.TIME_S.name]) / args.min_time_s)
    # Add another 5% for safety.
    iterations += iterations * 0.05
    iterations = int(iterations)
    program_result = run_program(program(args.cpu, args.matrix_file, iterations))

result[Stat.ITERATIONS.name] = iterations

result |= program_result[0]
print(program_result[1], file=sys.stderr)

result[Stat.TIME_S_1KI.name] = (
        (result[Stat.TIME_S.name] / result[Stat.ITERATIONS.name]) * 1000
)

if args.power:
    result[Stat.BASELINE_TIME_S.name] = args.baseline_time_s
    result[Stat.BASELINE_DELAY_S.name] = args.baseline_delay_s

    # Baseline
    time.sleep(args.baseline_delay_s)
    baseline_list = baseline_power(args.cpu, args.baseline_time_s)
    if args.debug:
        print(baseline_list)
    assert(len(baseline_list) == args.baseline_time_s)

    # Power Collection
    power_process = subprocess.run(
            ['./power.sh', '-1'] + program(args.cpu, args.matrix_file, result[Stat.ITERATIONS.name]),
            stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
    power_process.check_returncode()

    power_list = [float(x)
            for x in power_process.stdout.strip().split('\n')]
    power_time_s = json.loads(power_process.stderr)[Stat.TIME_S.name]
    if args.debug:
        print(power_list)
        print(power_time_s)
        
    if args.cpu == Cpu.ALTRA:
        # Trapezoidal Rule across the last (s) power recordings.
        result[Stat.J.name] = trapezoidal_rule(
                power_list, power_time_s)
    elif args.cpu == Cpu.EPYC_7313P or args.cpu == Cpu.XEON_4216:
        result[Stat.J.name] = power_list[0] * power_time_s 

    result[Stat.W.name] = result[Stat.J.name] / power_time_s

    if args.debug:
        print(result)
        #print(len(result['power']))
        #print(sum(result['power']) / len(result['power']))

    # Baseline
    time.sleep(args.baseline_delay_s)
    baseline_list += baseline_power(args.cpu, args.baseline_time_s)
    if args.debug:
        print(baseline_list)
    assert(len(baseline_list) / 2 == args.baseline_time_s)

    baseline_joules = (
        trapezoidal_rule(
            baseline_list[:args.baseline_time_s],
            args.baseline_time_s) +
        trapezoidal_rule(
            baseline_list[args.baseline_time_s:],
            args.baseline_time_s)
    )
    baseline_wattage = baseline_joules / (args.baseline_time_s * 2)
    
    result[Stat.J_1KI.name] = (
            (result[Stat.J.name] / result[Stat.ITERATIONS.name]) * 1000
    )
    result[Stat.W_1KI.name] = (
            (result[Stat.W.name] / result[Stat.ITERATIONS.name]) * 1000
    )
    result[Stat.W_D.name] = result[Stat.W.name] - baseline_wattage
    result[Stat.J_D.name] = result[Stat.W_D.name] * power_time_s
    result[Stat.W_D_1KI.name] = (
            (result[Stat.W_D.name] / result[Stat.ITERATIONS.name]) * 1000
    )
    result[Stat.J_D_1KI.name] = (
            (result[Stat.W_D_1KI.name] / result[Stat.ITERATIONS.name]) * 1000
    )

    if args.debug:
        print(result)

print(json.dumps(result))
#if args.perf:
#    for perf_arg in perf_args[args.cpu]:
#        output = run_program(perf + perf_arg + program[args.cpu])[1]
#        print(output, file=sys.stderr)
#        result = result | data_stat.parse_output(output, args.cpu)
#        if args.debug:
#            print(result)
#
#    result = result | data_stat.derive_stats(result)
#
#    if args.debug:
#        print(result)


#arch = subprocess.run(['uname', '-m'], stdout=subprocess.PIPE, text=True).stdout.strip()
#baseline = subprocess.run(
#        ['./power.sh', args.baseline_time_s],
#        stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
#print(baseline)
#for line in baseline.stdout.split('\n'):
#    print("line")
#    print(line)

#os.path.basename(args.matrix_file)

#subprocess.run(
#    ['apptainer', 'run', 'pytorch-altra.sif', '-c',
#        f'"numactl --cpunodebind=0 --membind=0 python spmv.py {args.matrix_file} {args.iterations}"'
#    ],
#    stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)