opentuner_example

opentuner_example

代码

#!/usr/bin/env python
#
# Optimize blocksize of apps/mmm_block.cpp
#
# This is an extremely simplified version meant only for tutorials
#
from __future__ import print_function

import opentuner
from opentuner import ConfigurationManipulator
from opentuner import IntegerParameter
from opentuner import MeasurementInterface
from opentuner import Result

#GccFlagsTuner 类的实例,它使用 opentuner 调整指定的参数
class GccFlagsTuner(MeasurementInterface):
	#定义搜索空间
    def manipulator(self):
        """
        Define the search space by creating a
        ConfigurationManipulator
        """
        manipulator = ConfigurationManipulator()
        manipulator.add_parameter(
            IntegerParameter('BLOCK_SIZE', 1, 20)) #搜索空间
        return manipulator

    #在给定配置下运行opentuner,并返回在此配置下计算出的性能
    def run(self, desired_result, input, limit):
        """
        Compile and run a given configuration then
        return performance
        """
        cfg = desired_result.configuration.data
        #print(cfg)
		
        ##gcc_cmd是命令行编译需要执行的语句
        gcc_cmd = 'g++ mmm_block.cpp '
        gcc_cmd += ' -D{0}={1}'.format('BLOCK_SIZE', cfg['BLOCK_SIZE'])
        gcc_cmd += ' -o ./tmp.bin'

        compile_result = self.call_program(gcc_cmd)
        #print(compile_result)
        #windows下值为0,linux下值为1
        assert compile_result['returncode'] == 0

        run_cmd = './tmp.bin'

        run_result = self.call_program(run_cmd)
        #print("hello world")
        #print(run_result)
        #windows下值为0,linux下值为1
        assert run_result['returncode'] == 0
        return Result(time=run_result['time'])
	#将找到的最佳 blockSize 参数的 json 字典保存到文件 mmm_final_config.json
    def save_final_config(self, configuration):
        """called at the end of tuning"""
        print("Optimal block size written to mmm_final_config.json:", configuration.data)
        self.manipulator().save_to_file(configuration.data,
                                        'mmm_final_config.json')


if __name__ == '__main__':
    #argparser是一个命令行解析器对象,它并不为空
    argparser = opentuner.default_argparser()
    #argparser.parse_args()是进行解析
    GccFlagsTuner.main(argparser.parse_args())

arg是什么

在数学里arg是解

.bin文件

binary,二进制文件

argparse模块

可以让人轻松编写用户友好的命令行接口

cfg的内部数据

image-20220517213102553

self.call_program(gcc_cmd)的返回值

compile_result的内部数据

run_result的内部数据

函数run的返回值

是一个对象

运行方法

Run the following command to autotune our program(The --no-dups flag hides warnings about duplicate results and the --stop-after parameter specifies that we are running opentuner for a maximum of 30 seconds):

python mmm_tuner.py --no-dups --stop-after=30

gccflags

重点

extract_working_params

重点在于对gcc参数的理解

编译器标志:gcc_flags

编译选项: gcc_params

example:

gcc_flags: asynchronous-unwind-tables

gcc -S -fno-asynchronous-unwind-tables execFunc.c

(52条消息) g++的英文版使用说明和选项_Lionel_Coder的博客-CSDN博客

在如上的博客里可以看出,gcc_flags和gcc_params都是如下的语法格式

Usage: g++ [options] file... Options:

PeteBricks

PetaBricks 是一种隐式并行编程语言,它的编译器对一个应用的多个算法版本进行多种实现,提供算法的多种选择是这么语言的基本结构,编译器可以根据算法的多种实现进行细粒度的调优。

example1

0.75版本

#!/usr/bin/env python
from __future__ import division
from __future__ import print_function
from builtins import map
from builtins import filter
from builtins import range
from past.utils import old_div

import math
import argparse
import ast
import collections
import json
import logging
import opentuner
import os
import random
import re
import shutil
import subprocess
import sys

from opentuner.resultsdb.models import Result, TuningRun
from opentuner.search import manipulator

FLAGS_WORKING_CACHE_FILE = 'cc_flags.json'
PARAMS_DEFAULTS_CACHE_FILE = 'cc_param_defaults.json'
PARAMS_DEF_PATH = '~/gcc-4.9.0/gcc/params.def'
PARAMS_WORKING_CACHE_FILE = 'cc_params.json'

log = logging.getLogger('gccflags')

argparser = argparse.ArgumentParser(parents=opentuner.argparsers())
argparser.add_argument('--source', default='tsp_ga.cpp',help='source file to compile')
argparser.add_argument('--compile-template',
                       default='{cc} {source} -o {output} -lpthread {flags}',
                       help='command to compile {source} into {output} with'
                            ' {flags}')
argparser.add_argument('--compile-limit', type=float, default=30,
                       help='kill gcc if it runs more than {default} sec')
argparser.add_argument('--scaler', type=int, default=4,
                       help='by what factor to try increasing parameters')
argparser.add_argument('--cc', default='g++', help='g++ or gcc')
argparser.add_argument('--output', default='./tmp.bin',
                       help='temporary file for compiler to write to')
argparser.add_argument('--debug', action='store_true',
                       help='on gcc errors try to find minimal set '
                            'of args to reproduce error')
argparser.add_argument('--force-killall', action='store_true',
                       help='killall cc1plus before each collection')
argparser.add_argument('--memory-limit', default=1024 ** 3, type=int,
                       help='memory limit for child process')
argparser.add_argument('--no-cached-flags', action='store_true',
                       help='regenerate the lists of legal flags each time')
argparser.add_argument('--flags-histogram', action='store_true',
                       help='print out a histogram of flags')
argparser.add_argument('--flag-importance',
                       help='Test the importance of different flags from a '
                            'given json file.')


class GccFlagsTuner(opentuner.measurement.MeasurementInterface):
    def __init__(self, *pargs, **kwargs):
        super(GccFlagsTuner, self).__init__(program_name=args.source, *pargs,
                                            **kwargs)
        self.gcc_version =self.extract_gcc_version()
        self.cc_flags = self.extract_working_flags()
        self.cc_param_defaults = self.extract_param_defaults()
        self.cc_params = self.extract_working_params()

        # these bugs are hardcoded for now
        # sets of options which causes gcc to barf
        if True:
            # These bugs were for gcc 4.7 on ubuntu
            self.cc_bugs = (['-fipa-matrix-reorg', '-fwhole-program'],
                            ['-fno-tree-coalesce-inlined-vars'],
                            ['-fno-inline-atomics'],
                            ['-ftoplevel-reorder', '-fno-unit-at-a-time'])
        else:
            # Bugs for gcc 4.9 (work in progress, incomplete list)
            self.cc_bugs = (['-ftoplevel-reorder', '-fno-unit-at-a-time'],)

        self.result_list = {}
        self.parallel_compile = True
        try:
            os.stat('./tmp')
        except OSError:
            os.mkdir('./tmp')
        self.run_baselines()

    def run_baselines(self):
        log.info("baseline perfs -O0=%.4f -O1=%.4f -O2=%.4f -O3=%.4f",
                 *[self.run_with_flags(['-O%d' % i], None).time
                   for i in range(4)])

    def extract_gcc_version(self):
        m = re.search(r'([0-9]+)[.]([0-9]+)[.]([0-9]+)', (subprocess.check_output([
            self.args.cc, '--version'])).decode('utf-8'))
        if m:
            gcc_version = tuple(map(int, m.group(1, 2, 3)))
        else:
            gcc_version = None
        log.debug('gcc version %s', gcc_version)
        return gcc_version

    def extract_working_flags(self):
        """
        Figure out which gcc flags work (don't cause gcc to barf) by running
        each one.
        """
        if os.path.isfile(FLAGS_WORKING_CACHE_FILE) and not args.no_cached_flags:
            # use cached version
            found_cc_flags = json.load(open(FLAGS_WORKING_CACHE_FILE))
        else:
            # extract flags from --help=optimizers
            optimizers, err = subprocess.Popen([self.args.cc, '--help=optimizers'],
                                               stdout=subprocess.PIPE).communicate()
            found_cc_flags = re.findall(r'^  (-f[a-z0-9-]+) ', optimizers,
                                        re.MULTILINE)
            log.info('Determining which of %s possible gcc flags work',
                     len(found_cc_flags))
            found_cc_flags = list(filter(self.check_if_flag_works, found_cc_flags))
            json.dump(found_cc_flags, open(FLAGS_WORKING_CACHE_FILE, 'w'))
        return found_cc_flags

    def extract_param_defaults(self):
        """
        Get the default, minimum, and maximum for each gcc parameter.
        Requires source code for gcc to be in your home directory.
        This example ships with a cached version so it does not require source.
        """
        if os.path.isfile(PARAMS_DEFAULTS_CACHE_FILE) and not args.no_cached_flags:
            # use cached version
            param_defaults = json.load(open(PARAMS_DEFAULTS_CACHE_FILE))
        else:
            # default values of params need to be extracted from source code,
            # since they are not in --help
            param_defaults = dict()
            params_def = open(os.path.expanduser(PARAMS_DEF_PATH)).read()
            for m in re.finditer(r'DEFPARAM *\((([^")]|"[^"]*")*)\)', params_def):
                param_def_str = (m.group(1)
                                 #  Hacks!!!
                                 .replace('GGC_MIN_EXPAND_DEFAULT', '30')
                                 .replace('GGC_MIN_HEAPSIZE_DEFAULT', '4096')
                                 .replace('50 * 1024 * 1024', '52428800'))
                try:
                    name, desc, default, param_min, param_max = ast.literal_eval(
                        '[' + param_def_str.split(',', 1)[1] + ']')
                    param_defaults[name] = {'default': default,
                                            'min': param_min,
                                            'max': param_max}
                except:
                    log.exception("error with %s", param_def_str)
            json.dump(param_defaults, open(PARAMS_DEFAULTS_CACHE_FILE, 'w'))
        return param_defaults

    def extract_working_params(self):
        """
        Figure out which gcc params work (don't cause gcc to barf) by running
        each one to test.
        """
        params, err = subprocess.Popen(
            [self.args.cc, '--help=params'], stdout=subprocess.PIPE).communicate()
        all_params = re.findall(r'^  ([a-z0-9-]+) ', params, re.MULTILINE)
        all_params = sorted(set(all_params) &
                            set(self.cc_param_defaults.keys()))
        if os.path.isfile(PARAMS_WORKING_CACHE_FILE) and not args.no_cached_flags:
            # use cached version
            return json.load(open(PARAMS_WORKING_CACHE_FILE))
        else:
            log.info('Determining which of %s possible gcc params work',
                     len(all_params))
            working_params = []
            for param in all_params:
                if self.check_if_flag_works('--param={}={}'.format(
                        param, self.cc_param_defaults[param]['default'])):
                    working_params.append(param)
            json.dump(working_params, open(PARAMS_WORKING_CACHE_FILE, 'w'))
            return working_params

    def check_if_flag_works(self, flag, try_inverted=True):
        cmd = args.compile_template.format(source=args.source, output=args.output,
                                           flags=flag, cc=args.cc)
        compile_result = self.call_program(cmd, limit=args.compile_limit)
        if compile_result['returncode'] != 0:
            log.warning("removing flag %s because it results in compile error", flag)
            return False
        if 'warning: this target' in compile_result['stderr']:
            log.warning("removing flag %s because not supported by target", flag)
            return False
        if 'has been renamed' in compile_result['stderr']:
            log.warning("removing flag %s because renamed", flag)
            return False
        if try_inverted and flag[:2] == '-f':
            if not self.check_if_flag_works(invert_gcc_flag(flag),
                                            try_inverted=False):
                log.warning("Odd... %s works but %s does not", flag,
                            invert_gcc_flag(flag))
                return False
        return True

    def manipulator(self):
        m = manipulator.ConfigurationManipulator()
        m.add_parameter(manipulator.IntegerParameter('-O', 0, 3))
        for flag in self.cc_flags:
            m.add_parameter(manipulator.EnumParameter(flag, ['on', 'off', 'default']))
        for param in self.cc_params:
            defaults = self.cc_param_defaults[param]
            if defaults['max'] <= defaults['min']:
                defaults['max'] = float('inf')
            defaults['max'] = min(defaults['max'],
                                  max(1, defaults['default']) * args.scaler)
            defaults['min'] = max(defaults['min'],
                                  old_div(max(1, defaults['default']), args.scaler))

            if param == 'l1-cache-line-size':
                # gcc requires this to be a power of two or it internal errors
                m.add_parameter(manipulator.PowerOfTwoParameter(param, 4, 256))
            elif defaults['max'] > 128:
                m.add_parameter(manipulator.LogIntegerParameter(
                    param, defaults['min'], defaults['max']))
            else:
                m.add_parameter(manipulator.IntegerParameter(
                    param, defaults['min'], defaults['max']))

        return m

    def cfg_to_flags(self, cfg):
        flags = ['-O%d' % cfg['-O']]
        for flag in self.cc_flags:
            if cfg[flag] == 'on':
                flags.append(flag)
            elif cfg[flag] == 'off':
                flags.append(invert_gcc_flag(flag))

        for param in self.cc_params:
            flags.append('--param=%s=%d' % (param, cfg[param]))

        # workaround sets of flags that trigger compiler crashes/hangs
        for bugset in self.cc_bugs:
            if len(set(bugset) & set(flags)) == len(bugset):
                flags.remove(bugset[-1])
        return flags

    def make_command(self, cfg):
        return args.compile_template.format(source=args.source, output=args.output,
                                            flags=' '.join(self.cfg_to_flags(cfg)),
                                            cc=args.cc)

    def get_tmpdir(self, result_id):
        return './tmp/%d' % result_id

    def cleanup(self, result_id):
        tmp_dir = self.get_tmpdir(result_id)
        shutil.rmtree(tmp_dir)

    def compile_and_run(self, desired_result, input, limit):
        cfg = desired_result.configuration.data
        compile_result = self.compile(cfg, 0)
        return self.run_precompiled(desired_result, input, limit, compile_result, 0)

    compile_results = {'ok': 0, 'timeout': 1, 'error': 2}

    def run_precompiled(self, desired_result, input, limit, compile_result,
                        result_id):
        if self.args.force_killall:
            os.system('killall -9 cc1plus 2>/dev/null')
        # Make sure compile was successful
        if compile_result == self.compile_results['timeout']:
            return Result(state='TIMEOUT', time=float('inf'))
        elif compile_result == self.compile_results['error']:
            return Result(state='ERROR', time=float('inf'))

        tmp_dir = self.get_tmpdir(result_id)
        output_dir = '%s/%s' % (tmp_dir, args.output)
        try:
            run_result = self.call_program([output_dir], limit=limit,
                                           memory_limit=args.memory_limit)
        except OSError:
            return Result(state='ERROR', time=float('inf'))

        if run_result['returncode'] != 0:
            if run_result['timeout']:
                return Result(state='TIMEOUT', time=float('inf'))
            else:
                log.error('program error')
                return Result(state='ERROR', time=float('inf'))

        return Result(time=run_result['time'])

    def debug_gcc_error(self, flags):
        def fails(subflags):
            cmd = args.compile_template.format(source=args.source, output=args.output,
                                               flags=' '.join(subflags),
                                               cc=args.cc)
            compile_result = self.call_program(cmd, limit=args.compile_limit)
            return compile_result['returncode'] != 0

        if self.args.debug:
            while len(flags) > 8:
                log.error("compile error with %d flags, diagnosing...", len(flags))
                tmpflags = [x for x in flags if random.choice((True, False))]
                if fails(tmpflags):
                    flags = tmpflags

            # linear scan
            minimal_flags = []
            for i in range(len(flags)):
                tmpflags = minimal_flags + flags[i + 1:]
                if not fails(tmpflags):
                    minimal_flags.append(flags[i])
            log.error("compiler crashes/hangs with flags: %s", minimal_flags)

    def compile(self, config_data, result_id):
        flags = self.cfg_to_flags(config_data)
        return self.compile_with_flags(flags, result_id)

    def compile_with_flags(self, flags, result_id):
        tmp_dir = self.get_tmpdir(result_id)
        try:
            os.stat(tmp_dir)
        except OSError:
            os.mkdir(tmp_dir)
        output_dir = '%s/%s' % (tmp_dir, args.output)
        cmd = args.compile_template.format(source=args.source, output=output_dir,
                                           flags=' '.join(flags),
                                           cc=args.cc)

        compile_result = self.call_program(cmd, limit=args.compile_limit,
                                           memory_limit=args.memory_limit)
        if compile_result['returncode'] != 0:
            if compile_result['timeout']:
                log.warning("gcc timeout")
                return self.compile_results['timeout']
            else:
                log.warning("gcc error %s", compile_result['stderr'])
                self.debug_gcc_error(flags)
                return self.compile_results['error']
        return self.compile_results['ok']

    def run_with_flags(self, flags, limit):
        return self.run_precompiled(None, None, limit,
                                    self.compile_with_flags(flags, 0), 0)

    def save_final_config(self, configuration):
        """called at the end of tuning"""
        print("Best flags written to gccflags_final_config.{json,cmd}")
        self.manipulator().save_to_file(configuration.data,
                                        'gccflags_final_config.json')
        with open('gccflags_final_config.cmd', 'w') as fd:
            fd.write(self.make_command(configuration.data))

    def flags_histogram(self, session):
        counter = collections.Counter()
        q = session.query(TuningRun).filter_by(state='COMPLETE')
        total = q.count()
        for tr in q:
            print(tr.program.name)
            for flag in self.cfg_to_flags(tr.final_config.data):
                counter[flag] += old_div(1.0, total)
        print(counter.most_common(20))

    def flag_importance(self):
        """
        Test the importance of each flag by measuring the performance with that
        flag removed.  Print out a table for paper
        """
        with open(self.args.flag_importance) as fd:
            best_cfg = json.load(fd)
        flags = self.cfg_to_flags(best_cfg)
        counter = collections.Counter()
        baseline_time = self.flags_mean_time(flags)
        for flag in flags[1:]:
            delta_flags = [f for f in flags if f != flag]
            flag_time = self.flags_mean_time(delta_flags)
            impact = max(0.0, flag_time - baseline_time)
            if math.isinf(impact):
                impact = 0.0
            counter[flag] = impact
            print(flag, '{:.4f}'.format(impact))
        total_impact = sum(counter.values())
        remaining_impact = total_impact
        print(r'\bf Flag & \bf Importance \\\hline')
        for flag, impact in counter.most_common(20):
            print(r'{} & {:.1f}\% \\\hline'.format(flag, 100.0 * impact / total_impact))
            remaining_impact -= impact
        print(r'{} other flags & {:.1f}% \\\hline'.format(
            len(flags) - 20, 100.0 * remaining_impact / total_impact))

    def flags_mean_time(self, flags, trials=10):
        precompiled = self.compile_with_flags(flags, 0)
        total = 0.0
        for _ in range(trials):
            total += self.run_precompiled(None, None, None, precompiled, 0).time
        return old_div(total, trials)

    def prefix_hook(self, session):
        if self.args.flags_histogram:
            self.flags_histogram(session)
            sys.exit(0)
        if self.args.flag_importance:
            self.flag_importance()
            sys.exit(0)


def invert_gcc_flag(flag):
    assert flag[:2] == '-f'
    if flag[2:5] != 'no-':
        return '-fno-' + flag[2:]
    return '-f' + flag[5:]


if __name__ == '__main__':
    opentuner.init_logging()
    args = argparser.parse_args()
    GccFlagsTuner.main(args)

可以执行的example1的代码

#!/usr/bin/env python
from __future__ import division
from __future__ import print_function
from builtins import map
from builtins import filter
from builtins import range
from past.utils import old_div

import math
import argparse
import ast
import collections
import json
import logging
import opentuner
import os
import random
import re
import shutil
import subprocess
import sys

from opentuner.resultsdb.models import Result, TuningRun
from opentuner.search import manipulator

FLAGS_WORKING_CACHE_FILE = 'cc_flags.json'
PARAMS_DEFAULTS_CACHE_FILE = 'cc_param_defaults.json'
PARAMS_DEF_PATH = '~/gcc-4.9.0/gcc/params.def'
PARAMS_WORKING_CACHE_FILE = 'cc_params.json'

log = logging.getLogger('gccflags')

argparser = argparse.ArgumentParser(parents=opentuner.argparsers())
argparser.add_argument('--source', default='tsp_ga.cpp',help='source file to compile')
argparser.add_argument('--compile-template',
                       default='{cc} {source} -o {output} -lpthread {flags}',
                       help='command to compile {source} into {output} with'
                            ' {flags}')
argparser.add_argument('--compile-limit', type=float, default=30,
                       help='kill gcc if it runs more than {default} sec')
argparser.add_argument('--scaler', type=int, default=4,
                       help='by what factor to try increasing parameters')
argparser.add_argument('--cc', default='g++', help='g++ or gcc')
argparser.add_argument('--output', default='./tmp.bin',
                       help='temporary file for compiler to write to')
argparser.add_argument('--debug', action='store_true',
                       help='on gcc errors try to find minimal set '
                            'of args to reproduce error')
argparser.add_argument('--force-killall', action='store_true',
                       help='killall cc1plus before each collection')
argparser.add_argument('--memory-limit', default=1024 ** 3, type=int,
                       help='memory limit for child process')
argparser.add_argument('--no-cached-flags', action='store_true',
                       help='regenerate the lists of legal flags each time')
argparser.add_argument('--flags-histogram', action='store_true',
                       help='print out a histogram of flags')
argparser.add_argument('--flag-importance',
                       help='Test the importance of different flags from a '
                            'given json file.')


class GccFlagsTuner(opentuner.measurement.MeasurementInterface):
    def __init__(self, *pargs, **kwargs):
        super(GccFlagsTuner, self).__init__(program_name=args.source, *pargs,
                                            **kwargs)
        self.gcc_version =self.extract_gcc_version()
        self.cc_flags = self.extract_working_flags()
        self.cc_param_defaults = self.extract_param_defaults()
        self.cc_params = self.extract_working_params()

        # these bugs are hardcoded for now
        # sets of options which causes gcc to barf
        if True:
            # These bugs were for gcc 4.7 on ubuntu
            self.cc_bugs = (['-fipa-matrix-reorg', '-fwhole-program'],
                            ['-fno-tree-coalesce-inlined-vars'],
                            ['-fno-inline-atomics'],
                            ['-ftoplevel-reorder', '-fno-unit-at-a-time'])
        else:
            # Bugs for gcc 4.9 (work in progress, incomplete list)
            self.cc_bugs = (['-ftoplevel-reorder', '-fno-unit-at-a-time'],)

        self.result_list = {}
        self.parallel_compile = True
        try:
            os.stat('./tmp')
        except OSError:
            os.mkdir('./tmp')
        self.run_baselines()

    def run_baselines(self):
        log.info("baseline perfs -O0=%.4f -O1=%.4f -O2=%.4f -O3=%.4f",
                 *[self.run_with_flags(['-O%d' % i], None).time
                   for i in range(4)])

    def extract_gcc_version(self):
        m = re.search(r'([0-9]+)[.]([0-9]+)[.]([0-9]+)', (subprocess.check_output([
            self.args.cc, '--version'])).decode('utf-8'))
        if m:
            gcc_version = tuple(map(int, m.group(1, 2, 3)))
        else:
            gcc_version = None
        log.debug('gcc version %s', gcc_version)
        return gcc_version

    def extract_working_flags(self):
        """
        Figure out which gcc flags work (don't cause gcc to barf) by running
        each one.
        """
        if os.path.isfile(FLAGS_WORKING_CACHE_FILE) and not args.no_cached_flags:
            # use cached version
            found_cc_flags = json.load(open(FLAGS_WORKING_CACHE_FILE))
        else:
            # extract flags from --help=optimizers
            optimizers, err = subprocess.Popen([self.args.cc, '--help=optimizers'],
                                               stdout=subprocess.PIPE).communicate()
            found_cc_flags = re.findall(r'^  (-f[a-z0-9-]+) ', optimizers.decode('utf-8'),
                                        re.MULTILINE)

            log.info('Determining which of %s possible gcc flags work',
                     len(found_cc_flags))
            b=''
            c =b.join(found_cc_flags).encode()
            found_cc_flags = list(filter(self.check_if_flag_works, c))
            json.dump(found_cc_flags, open(FLAGS_WORKING_CACHE_FILE, 'w'))
        return found_cc_flags

    def extract_param_defaults(self):
        """
        Get the default, minimum, and maximum for each gcc parameter.
        Requires source code for gcc to be in your home directory.
        This example ships with a cached version so it does not require source.
        """
        if os.path.isfile(PARAMS_DEFAULTS_CACHE_FILE) and not args.no_cached_flags:
            # use cached version
            param_defaults = json.load(open(PARAMS_DEFAULTS_CACHE_FILE))
        else:
            # default values of params need to be extracted from source code,
            # since they are not in --help
            param_defaults = dict()
            params_def = open(os.path.expanduser(PARAMS_DEF_PATH)).read()
            for m in re.finditer(r'DEFPARAM *\((([^")]|"[^"]*")*)\)', params_def):
                param_def_str = (m.group(1)
                                 #  Hacks!!!
                                 .replace('GGC_MIN_EXPAND_DEFAULT', '30')
                                 .replace('GGC_MIN_HEAPSIZE_DEFAULT', '4096')
                                 .replace('50 * 1024 * 1024', '52428800'))
                try:
                    name, desc, default, param_min, param_max = ast.literal_eval(
                        '[' + param_def_str.split(',', 1)[1] + ']')
                    param_defaults[name] = {'default': default,
                                            'min': param_min,
                                            'max': param_max}
                except:
                    log.exception("error with %s", param_def_str)
            json.dump(param_defaults, open(PARAMS_DEFAULTS_CACHE_FILE, 'w'))
        return param_defaults

    def extract_working_params(self):
        """
        Figure out which gcc params work (don't cause gcc to barf) by running
        each one to test.
        """
        params, err = subprocess.Popen(
            [self.args.cc, '--help=params'], stdout=subprocess.PIPE).communicate()
        all_params = re.findall(r'^  ([a-z0-9-]+) ', params.decode('utf-8'), re.MULTILINE)
        all_params = sorted(set(all_params) &
                            set(self.cc_param_defaults.keys()))
        if os.path.isfile(PARAMS_WORKING_CACHE_FILE) and not args.no_cached_flags:
            # use cached version
            return json.load(open(PARAMS_WORKING_CACHE_FILE))
        else:
            log.info('Determining which of %s possible gcc params work',
                     len(all_params))
            working_params = []
            for param in all_params:
                if self.check_if_flag_works('--param={}={}'.format(
                        param.encode(), self.cc_param_defaults[param]['default'])):
                    working_params.append(param)
            json.dump(working_params, open(PARAMS_WORKING_CACHE_FILE, 'w'))
            return working_params

    def check_if_flag_works(self, flag, try_inverted=True):
        cmd = args.compile_template.format(source=args.source, output=args.output,
                                           flags=flag, cc=args.cc)
        compile_result = self.call_program(cmd, limit=args.compile_limit)
        if compile_result['returncode'] != 0:
            log.warning("removing flag %s because it results in compile error", flag)
            return False
        if 'warning: this target' in compile_result['stderr']:
            log.warning("removing flag %s because not supported by target", flag)
            return False
        if 'has been renamed' in compile_result['stderr']:
            log.warning("removing flag %s because renamed", flag)
            return False
        if try_inverted and flag[:2] == '-f':
            if not self.check_if_flag_works(invert_gcc_flag(flag),
                                            try_inverted=False):
                log.warning("Odd... %s works but %s does not", flag,
                            invert_gcc_flag(flag))
                return False
        return True

    def manipulator(self):
        m = manipulator.ConfigurationManipulator()
        m.add_parameter(manipulator.IntegerParameter('-O', 0, 3))
        for flag in self.cc_flags:
            m.add_parameter(manipulator.EnumParameter(flag, ['on', 'off', 'default']))
        for param in self.cc_params:
            defaults = self.cc_param_defaults[param]
            if defaults['max'] <= defaults['min']:
                defaults['max'] = float('inf')
            defaults['max'] = min(defaults['max'],
                                  max(1, defaults['default']) * args.scaler)
            defaults['min'] = max(defaults['min'],
                                  old_div(max(1, defaults['default']), args.scaler))

            if param == 'l1-cache-line-size':
                # gcc requires this to be a power of two or it internal errors
                m.add_parameter(manipulator.PowerOfTwoParameter(param, 4, 256))
            elif defaults['max'] > 128:
                m.add_parameter(manipulator.LogIntegerParameter(
                    param, defaults['min'], defaults['max']))
            else:
                m.add_parameter(manipulator.IntegerParameter(
                    param, defaults['min'], defaults['max']))

        return m

    def cfg_to_flags(self, cfg):
        flags = ['-O%d' % cfg['-O']]
        for flag in self.cc_flags:
            if cfg[flag] == 'on':
                flags.append(flag)
            elif cfg[flag] == 'off':
                flags.append(invert_gcc_flag(flag))

        for param in self.cc_params:
            flags.append('--param=%s=%d' % (param, cfg[param]))

        # workaround sets of flags that trigger compiler crashes/hangs
        for bugset in self.cc_bugs:
            if len(set(bugset) & set(flags)) == len(bugset):
                flags.remove(bugset[-1])
        return flags

    def make_command(self, cfg):
        return args.compile_template.format(source=args.source, output=args.output,
                                            flags=' '.join(self.cfg_to_flags(cfg)),
                                            cc=args.cc)

    def get_tmpdir(self, result_id):
        return './tmp/%d' % result_id

    def cleanup(self, result_id):
        tmp_dir = self.get_tmpdir(result_id)
        shutil.rmtree(tmp_dir)

    def compile_and_run(self, desired_result, input, limit):
        cfg = desired_result.configuration.data
        compile_result = self.compile(cfg, 0)
        return self.run_precompiled(desired_result, input, limit, compile_result, 0)

    compile_results = {'ok': 0, 'timeout': 1, 'error': 2}

    def run_precompiled(self, desired_result, input, limit, compile_result,
                        result_id):
        if self.args.force_killall:
            os.system('killall -9 cc1plus 2>/dev/null')
        # Make sure compile was successful
        if compile_result == self.compile_results['timeout']:
            return Result(state='TIMEOUT', time=float('inf'))
        elif compile_result == self.compile_results['error']:
            return Result(state='ERROR', time=float('inf'))

        tmp_dir = self.get_tmpdir(result_id)
        output_dir = '%s/%s' % (tmp_dir, args.output)
        try:
            run_result = self.call_program([output_dir], limit=limit,
                                           memory_limit=args.memory_limit)
        except OSError:
            return Result(state='ERROR', time=float('inf'))

        if run_result['returncode'] != 0:
            if run_result['timeout']:
                return Result(state='TIMEOUT', time=float('inf'))
            else:
                log.error('program error')
                return Result(state='ERROR', time=float('inf'))

        return Result(time=run_result['time'])

    def debug_gcc_error(self, flags):
        def fails(subflags):
            cmd = args.compile_template.format(source=args.source, output=args.output,
                                               flags=' '.join(subflags),
                                               cc=args.cc)
            compile_result = self.call_program(cmd, limit=args.compile_limit)
            return compile_result['returncode'] != 0

        if self.args.debug:
            while len(flags) > 8:
                log.error("compile error with %d flags, diagnosing...", len(flags))
                tmpflags = [x for x in flags if random.choice((True, False))]
                if fails(tmpflags):
                    flags = tmpflags

            # linear scan
            minimal_flags = []
            for i in range(len(flags)):
                tmpflags = minimal_flags + flags[i + 1:]
                if not fails(tmpflags):
                    minimal_flags.append(flags[i])
            log.error("compiler crashes/hangs with flags: %s", minimal_flags)

    def compile(self, config_data, result_id):
        flags = self.cfg_to_flags(config_data)
        return self.compile_with_flags(flags, result_id)

    def compile_with_flags(self, flags, result_id):
        tmp_dir = self.get_tmpdir(result_id)
        try:
            os.stat(tmp_dir)
        except OSError:
            os.mkdir(tmp_dir)
        output_dir = '%s/%s' % (tmp_dir, args.output)
        cmd = args.compile_template.format(source=args.source, output=output_dir,
                                           flags=' '.join(flags),
                                           cc=args.cc)

        compile_result = self.call_program(cmd, limit=args.compile_limit,
                                           memory_limit=args.memory_limit)
        if compile_result['returncode'] != 0:
            if compile_result['timeout']:
                log.warning("gcc timeout")
                return self.compile_results['timeout']
            else:
                log.warning("gcc error %s", compile_result['stderr'])
                self.debug_gcc_error(flags)
                return self.compile_results['error']
        return self.compile_results['ok']

    def run_with_flags(self, flags, limit):
        return self.run_precompiled(None, None, limit,
                                    self.compile_with_flags(flags, 0), 0)

    def save_final_config(self, configuration):
        """called at the end of tuning"""
        print("Best flags written to gccflags_final_config.{json,cmd}")
        self.manipulator().save_to_file(configuration.data,
                                        'gccflags_final_config.json')
        with open('gccflags_final_config.cmd', 'w') as fd:
            fd.write(self.make_command(configuration.data))

    def flags_histogram(self, session):
        counter = collections.Counter()
        q = session.query(TuningRun).filter_by(state='COMPLETE')
        total = q.count()
        for tr in q:
            print(tr.program.name)
            for flag in self.cfg_to_flags(tr.final_config.data):
                counter[flag] += old_div(1.0, total)
        print(counter.most_common(20))

    def flag_importance(self):
        """
        Test the importance of each flag by measuring the performance with that
        flag removed.  Print out a table for paper
        """
        with open(self.args.flag_importance) as fd:
            best_cfg = json.load(fd)
        flags = self.cfg_to_flags(best_cfg)
        counter = collections.Counter()
        baseline_time = self.flags_mean_time(flags)
        for flag in flags[1:]:
            delta_flags = [f for f in flags if f != flag]
            flag_time = self.flags_mean_time(delta_flags)
            impact = max(0.0, flag_time - baseline_time)
            if math.isinf(impact):
                impact = 0.0
            counter[flag] = impact
            print(flag, '{:.4f}'.format(impact))
        total_impact = sum(counter.values())
        remaining_impact = total_impact
        print(r'\bf Flag & \bf Importance \\\hline')
        for flag, impact in counter.most_common(20):
            print(r'{} & {:.1f}\% \\\hline'.format(flag, 100.0 * impact / total_impact))
            remaining_impact -= impact
        print(r'{} other flags & {:.1f}% \\\hline'.format(
            len(flags) - 20, 100.0 * remaining_impact / total_impact))

    def flags_mean_time(self, flags, trials=10):
        precompiled = self.compile_with_flags(flags, 0)
        total = 0.0
        for _ in range(trials):
            total += self.run_precompiled(None, None, None, precompiled, 0).time
        return old_div(total, trials)

    def prefix_hook(self, session):
        if self.args.flags_histogram:
            self.flags_histogram(session)
            sys.exit(0)
        if self.args.flag_importance:
            self.flag_importance()
            sys.exit(0)


def invert_gcc_flag(flag):
    assert flag[:2] == '-f'
    if flag[2:5] != 'no-':
        return '-fno-' + flag[2:]
    return '-f' + flag[5:]


if __name__ == '__main__':
    opentuner.init_logging()
    args = argparser.parse_args()
    GccFlagsTuner.main(args)

想要显示importance

直接执行时显示 expected str, bytes or os.PathLike object, not NoneType

观察代码,重写了prefix_hook接口,接口里处理了importance

这是正确答案

#!/usr/bin/env python
from __future__ import division
from __future__ import print_function
from builtins import map
from builtins import filter
from builtins import range
from past.utils import old_div

import math
import argparse
import ast
import collections
import json
import logging
import opentuner
import os
import random
import re
import shutil
import subprocess
import sys

from opentuner.resultsdb.models import Result, TuningRun
from opentuner.search import manipulator

FLAGS_WORKING_CACHE_FILE = 'cc_flags.json'
PARAMS_DEFAULTS_CACHE_FILE = 'cc_param_defaults.json'
PARAMS_DEF_PATH = '~/gcc-4.9.0/gcc/params.def'
PARAMS_WORKING_CACHE_FILE = 'cc_params.json'

log = logging.getLogger('gccflags')

argparser = argparse.ArgumentParser(parents=opentuner.argparsers())
argparser.add_argument('source',help='source file to compile')
argparser.add_argument('--compile-template',
                       default='{cc} {source} -o {output} -lpthread {flags}',
                       help='command to compile {source} into {output} with'
                            ' {flags}')
argparser.add_argument('--compile-limit', type=float, default=30,
                       help='kill gcc if it runs more than {default} sec')
argparser.add_argument('--scaler', type=int, default=4,
                       help='by what factor to try increasing parameters')
argparser.add_argument('--cc', default='g++', help='g++ or gcc')
argparser.add_argument('--output', default='./tmp.bin',
                       help='temporary file for compiler to write to')
argparser.add_argument('--debug', action='store_true',
                       help='on gcc errors try to find minimal set '
                            'of args to reproduce error')
argparser.add_argument('--force-killall', action='store_true',
                       help='killall cc1plus before each collection')
argparser.add_argument('--memory-limit', default=1024 ** 3, type=int,
                       help='memory limit for child process')
argparser.add_argument('--no-cached-flags', action='store_true',
                       help='regenerate the lists of legal flags each time')
argparser.add_argument('--flags-histogram', action='store_true',
                       help='print out a histogram of flags')
argparser.add_argument('--flag-importance',
                       help='Test the importance of different flags from a '
                            'given json file.')


class GccFlagsTuner(opentuner.measurement.MeasurementInterface):
    def __init__(self, *pargs, **kwargs):
        super(GccFlagsTuner, self).__init__(program_name=args.source, *pargs,
                                            **kwargs)
        
        #self.flags_histogram()
        
        self.gcc_version =self.extract_gcc_version()
        self.cc_flags = self.extract_working_flags()
        self.cc_param_defaults = self.extract_param_defaults()
        #print(self.cc_param_defaults)
        self.cc_params = self.extract_working_params()

        # these bugs are hardcoded for now
        # sets of options which causes gcc to barf
        if True:
            # These bugs were for gcc 4.7 on ubuntu
            self.cc_bugs = (['-fipa-matrix-reorg', '-fwhole-program'],
                            ['-fno-tree-coalesce-inlined-vars'],
                            ['-fno-inline-atomics'],
                            ['-ftoplevel-reorder', '-fno-unit-at-a-time'])
        else:
            # Bugs for gcc 4.9 (work in progress, incomplete list)
            self.cc_bugs = (['-ftoplevel-reorder', '-fno-unit-at-a-time'],)

        self.result_list = {}
        self.parallel_compile = True
        try:
            os.stat('./tmp')
        except OSError:
            os.mkdir('./tmp')
        self.run_baselines()
        

    def run_baselines(self):
        log.info("baseline perfs -O0=%.4f -O1=%.4f -O2=%.4f -O3=%.4f",
                 *[self.run_with_flags(['-O%d' % i], None).time
                   for i in range(4)])

    def extract_gcc_version(self):
        m = re.search(r'([0-9]+)[.]([0-9]+)[.]([0-9]+)', (subprocess.check_output([
            self.args.cc, '--version'])).decode('utf-8'))
        if m:
            gcc_version = tuple(map(int, m.group(1, 2, 3)))
        else:
            gcc_version = None
        log.debug('gcc version %s', gcc_version)
        return gcc_version

    def extract_working_flags(self):
        """
        Figure out which gcc flags work (don't cause gcc to barf) by running
        each one.
        """
        if os.path.isfile(FLAGS_WORKING_CACHE_FILE) and not args.no_cached_flags:
            # use cached version
            found_cc_flags = json.load(open(FLAGS_WORKING_CACHE_FILE))
            ans=found_cc_flags
        else:
            # extract flags from --help=optimizers
            optimizers, err = subprocess.Popen([self.args.cc, '--help=optimizers'],
                                               stdout=subprocess.PIPE).communicate()
            # print("origin")
            # print(optimizers)
            found_cc_flags = re.findall(r'^  (-f[a-z0-9-]+) ', optimizers.decode('utf-8'),
                                        re.MULTILINE)
            
            log.info('Determining which of %s possible gcc flags work',
                     len(found_cc_flags))
            # b=''
            # c =b.join(found_cc_flags).encode()
            #found_cc_flags = list(filter(self.check_if_flag_works, found_cc_flags))
            ans=[]
            t=0
            for i in found_cc_flags:
                t+=1
                print(t)
                if(self.check_if_flag_works(i)):
                    ans.append(i)
            
            print(ans)
            # print("what is ccflag")
            # print(found_cc_flags)
            json.dump(ans, open(FLAGS_WORKING_CACHE_FILE, 'w'))
        return ans

    def extract_param_defaults(self):
        """
        Get the default, minimum, and maximum for each gcc parameter.
        Requires source code for gcc to be in your home directory.
        This example ships with a cached version so it does not require source.
        """
        if os.path.isfile(PARAMS_DEFAULTS_CACHE_FILE) and not args.no_cached_flags:
            # use cached version
            param_defaults = json.load(open(PARAMS_DEFAULTS_CACHE_FILE))
        else:
            # default values of params need to be extracted from source code,
            # since they are not in --help
            param_defaults = dict()
            params_def = open(os.path.expanduser(PARAMS_DEF_PATH)).read()
            for m in re.finditer(r'DEFPARAM *\((([^")]|"[^"]*")*)\)', params_def):
                param_def_str = (m.group(1)
                                 #  Hacks!!!
                                 .replace('GGC_MIN_EXPAND_DEFAULT', '30')
                                 .replace('GGC_MIN_HEAPSIZE_DEFAULT', '4096')
                                 .replace('50 * 1024 * 1024', '52428800'))
                try:
                    name, desc, default, param_min, param_max = ast.literal_eval(
                        '[' + param_def_str.split(',', 1)[1] + ']')
                    param_defaults[name] = {'default': default,
                                            'min': param_min,
                                            'max': param_max}
                except:
                    log.exception("error with %s", param_def_str)
            json.dump(param_defaults, open(PARAMS_DEFAULTS_CACHE_FILE, 'w'))
        return param_defaults

    def extract_working_params(self):
        """
        Figure out which gcc params work (don't cause gcc to barf) by running
        each one to test.
        """
        params, err = subprocess.Popen(
            [self.args.cc, '--help=params'], stdout=subprocess.PIPE).communicate()
        all_params = re.findall(r'^  ([a-z0-9-]+) ', params.decode('utf-8'), re.MULTILINE)
        all_params = sorted(set(all_params) &
                            set(self.cc_param_defaults.keys()))
        if os.path.isfile(PARAMS_WORKING_CACHE_FILE) and not args.no_cached_flags:
            # use cached version
            return json.load(open(PARAMS_WORKING_CACHE_FILE))
        else:
            log.info('Determining which of %s possible gcc params work',
                     len(all_params))
            working_params = []
            for param in all_params:
                if self.check_if_flag_works('--param={}={}'.format(
                        param.encode(), self.cc_param_defaults[param]['default'])):
                    working_params.append(param)
            json.dump(working_params, open(PARAMS_WORKING_CACHE_FILE, 'w'))
            return working_params

    def check_if_flag_works(self, flag, try_inverted=True):
        #flag=flag.encode()
        cmd = args.compile_template.format(source=args.source, output=args.output,
                                           flags=flag, cc=args.cc)
        compile_result = self.call_program(cmd, limit=args.compile_limit)
        #print(compile_result['stderr'])
        
        if compile_result['returncode'] != 0:
            log.warning("removing flag %s because it results in compile error", flag)
            return False
        
        if(b''!=compile_result['stderr']):   #证明有waring出现
            return False
        
        # if b'warning: this target' in compile_result['stderr']:
        #     log.warning("removing flag %s because not supported by target", flag) #%s的乱码
        #     return False
        # if b'has been renamed' in compile_result['stderr']:
        #     log.warning("removing flag %s because renamed", flag)  #%s的乱码
        #     return False
        if try_inverted and flag[:2] == '-f':
            if not self.check_if_flag_works(invert_gcc_flag(flag),
                                            try_inverted=False):
                log.warning("Odd... %s works but %s does not", flag,
                            invert_gcc_flag(flag))
                return False
        return True

    def manipulator(self):
        m = manipulator.ConfigurationManipulator()
        m.add_parameter(manipulator.IntegerParameter('-O', 0, 3))
        for flag in self.cc_flags:
            m.add_parameter(manipulator.EnumParameter(flag, ['on', 'off', 'default']))
        for param in self.cc_params:
            defaults = self.cc_param_defaults[param]
            if defaults['max'] <= defaults['min']:
                defaults['max'] = float('inf')
            defaults['max'] = min(defaults['max'],
                                  max(1, defaults['default']) * args.scaler)
            defaults['min'] = max(defaults['min'],
                                  old_div(max(1, defaults['default']), args.scaler))

            if param == 'l1-cache-line-size':
                # gcc requires this to be a power of two or it internal errors
                m.add_parameter(manipulator.PowerOfTwoParameter(param, 4, 256))
            elif defaults['max'] > 128:
                m.add_parameter(manipulator.LogIntegerParameter(
                    param, defaults['min'], defaults['max']))
            else:
                m.add_parameter(manipulator.IntegerParameter(
                    param, defaults['min'], defaults['max']))

        return m

    def cfg_to_flags(self, cfg):
        flags = ['-O%d' % cfg['-O']]
        for flag in self.cc_flags:
            if cfg[flag] == 'on':
                flags.append(flag)
            elif cfg[flag] == 'off':
                flags.append(invert_gcc_flag(flag))

        for param in self.cc_params:
            flags.append('--param=%s=%d' % (param, cfg[param]))

        # workaround sets of flags that trigger compiler crashes/hangs
        for bugset in self.cc_bugs:
            if len(set(bugset) & set(flags)) == len(bugset):
                flags.remove(bugset[-1])
        return flags

    def make_command(self, cfg):
        return args.compile_template.format(source=args.source, output=args.output,
                                            flags=' '.join(self.cfg_to_flags(cfg)),
                                            cc=args.cc)

    def get_tmpdir(self, result_id):
        return './tmp/%d' % result_id

    def cleanup(self, result_id):
        tmp_dir = self.get_tmpdir(result_id)
        shutil.rmtree(tmp_dir)

    def compile_and_run(self, desired_result, input, limit):
        cfg = desired_result.configuration.data
        compile_result = self.compile(cfg, 0)
        return self.run_precompiled(desired_result, input, limit, compile_result, 0)

    compile_results = {'ok': 0, 'timeout': 1, 'error': 2}

    def run_precompiled(self, desired_result, input, limit, compile_result,
                        result_id):
        if self.args.force_killall:
            os.system('killall -9 cc1plus 2>/dev/null')
        # Make sure compile was successful
        if compile_result == self.compile_results['timeout']:
            return Result(state='TIMEOUT', time=float('inf'))
        elif compile_result == self.compile_results['error']:
            return Result(state='ERROR', time=float('inf'))

        tmp_dir = self.get_tmpdir(result_id)
        output_dir = '%s/%s' % (tmp_dir, args.output)
        try:
            run_result = self.call_program([output_dir], limit=limit,
                                           memory_limit=args.memory_limit)
        except OSError:
            return Result(state='ERROR', time=float('inf'))

        if run_result['returncode'] != 0:
            if run_result['timeout']:
                return Result(state='TIMEOUT', time=float('inf'))
            else:
                log.error('program error')
                return Result(state='ERROR', time=float('inf'))

        return Result(time=run_result['time'])

    def debug_gcc_error(self, flags):
        def fails(subflags):
            cmd = args.compile_template.format(source=args.source, output=args.output,
                                               flags=' '.join(subflags),
                                               cc=args.cc)
            compile_result = self.call_program(cmd, limit=args.compile_limit)
            return compile_result['returncode'] != 0

        if self.args.debug:
            while len(flags) > 8:
                log.error("compile error with %d flags, diagnosing...", len(flags))
                tmpflags = [x for x in flags if random.choice((True, False))]
                if fails(tmpflags):
                    flags = tmpflags

            # linear scan
            minimal_flags = []
            for i in range(len(flags)):
                tmpflags = minimal_flags + flags[i + 1:]
                if not fails(tmpflags):
                    minimal_flags.append(flags[i])
            log.error("compiler crashes/hangs with flags: %s", minimal_flags)

    def compile(self, config_data, result_id):
        flags = self.cfg_to_flags(config_data)
        return self.compile_with_flags(flags, result_id)

    def compile_with_flags(self, flags, result_id):
        tmp_dir = self.get_tmpdir(result_id)
        try:
            os.stat(tmp_dir)
        except OSError:
            os.mkdir(tmp_dir)
        output_dir = '%s/%s' % (tmp_dir, args.output)
        cmd = args.compile_template.format(source=args.source, output=output_dir,
                                           flags=' '.join(flags),
                                           cc=args.cc)

        compile_result = self.call_program(cmd, limit=args.compile_limit,
                                           memory_limit=args.memory_limit)
        if compile_result['returncode'] != 0:
            if compile_result['timeout']:
                log.warning("gcc timeout")
                return self.compile_results['timeout']
            else:
                log.warning("gcc error %s", compile_result['stderr'])
                self.debug_gcc_error(flags)
                return self.compile_results['error']
        return self.compile_results['ok']

    def run_with_flags(self, flags, limit):
        return self.run_precompiled(None, None, limit,
                                    self.compile_with_flags(flags, 0), 0)

    def save_final_config(self, configuration):
        """called at the end of tuning"""
        print("Best flags written to gccflags_final_config.{json,cmd}")
        self.manipulator().save_to_file(configuration.data,
                                        'gccflags_final_config.json')
        with open('gccflags_final_config.cmd', 'w') as fd:
            fd.write(self.make_command(configuration.data))
                

    def flags_histogram(self, session):
        counter = collections.Counter()
        q = session.query(TuningRun).filter_by(state='COMPLETE')
        total = q.count()
        print("begin")
        for tr in q:
            print(tr.program.name)
            for flag in self.cfg_to_flags(tr.final_config.data):
                counter[flag] += old_div(1.0, total)
        print(counter.most_common(20))
        print("over")

    def flag_importance(self):
        """
        Test the importance of each flag by measuring the performance with that
        flag removed.  Print out a table for paper
        """
        print("hello world2")
        #with open(self.args.flag_importance) as fd:
        #flags = self.cfg_to_flags(best_cfg)
        flags=self.cc_flags
        #print(flags)
        print("print over")
        counter = collections.Counter()
        baseline_time = self.flags_mean_time(flags)
        for flag in flags[1:]:
            delta_flags = [f for f in flags if f != flag]
            flag_time = self.flags_mean_time(delta_flags)
            impact = max(0.0, flag_time - baseline_time)
            if math.isinf(impact):
                impact = 0.0
            counter[flag] = impact
            #print(flag, '{:.4f}'.format(impact))
        total_impact = sum(counter.values())
        print("totalimpaci is %d",total_impact)
        remaining_impact = total_impact
        print(r'\bf Flag & \bf Importance \\\hline')
        for flag, impact in counter.most_common(20):
            print(r'{} & {:.1f}\% \\\hline'.format(flag, 100.0 * impact / total_impact))
            remaining_impact -= impact
        print(r'{} other flags & {:.1f}% \\\hline'.format(
            len(flags) - 20, 100.0 * remaining_impact / total_impact))

    def flags_mean_time(self, flags, trials=10):
        precompiled = self.compile_with_flags(flags, 0)
        total = 0.0
        for _ in range(trials):
            total += self.run_precompiled(None, None, None, precompiled, 0).time
        return old_div(total, trials)

    def prefix_hook(self, session):
        #print(self.args.flags_histogram)
        print("here")
        if self.args.flags_histogram:
            print("coming")
            self.flags_histogram(session)
            print("over")
            sys.exit(0)
        if self.args.flag_importance:
            print("cominging")
            self.flag_importance()
            sys.exit(0)


def invert_gcc_flag(flag):
    assert flag[:2] == '-f'
    if flag[2:5] != 'no-':
        return '-fno-' + flag[2:]
    return '-f' + flag[5:]


if __name__ == '__main__':
    opentuner.init_logging()
    args = argparser.parse_args()
    #args.flag_importance='gccflags_final_config.json' #true或fause
    #args.flags_histogram=True
    #args.flag_importance=log
    GccFlagsTuner.main(args)

ray

图表中是中位数

ray优化后的结果

image-20220525085845238

图表

[外链图片转存失败,源站可能有防盗链机制,建议将图片保存下来直接上传(img-QW4uWQiZ-1663601565686)(https://raw.githubusercontent.com/theeeee6/test_clone/main/202209192328336.png)]

image-20220526124058509

mat

图表

image-20220525231102724

重要性

[外链图片转存失败,源站可能有防盗链机制,建议将图片保存下来直接上传(img-lypLFcyb-1663601565689)(https://raw.githubusercontent.com/theeeee6/test_clone/main/202209192328640.png)]

tsp

图表

image-20220525233826809

[外链图片转存失败,源站可能有防盗链机制,建议将图片保存下来直接上传(img-WKlIZfHP-1663601565692)(https://raw.githubusercontent.com/theeeee6/test_clone/main/202209192328058.png)]

example2 hpl

image-20220527171516887

搭建实验平台

GotoBLAS2、MPICH、HPL

安装成功

image-20220530010726741

  • 0
    点赞
  • 0
    收藏
    觉得还不错? 一键收藏
  • 0
    评论

“相关推荐”对你有帮助么?

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值