malloc_free.cc

// This file (malloc_free.cc) was created by Ron Rechenmacher <ron@fnal.gov> on
// Dec  2, 2015. "TERMS AND CONDITIONS" governing this file are in the README
// or COPYING file. If you do not have such a file, one can be obtained by
// contacting Ron or Fermi Lab in Batavia IL, 60510, phone: 630-840-3000.
// $RCSfile: malloc_free.cc,v $
static char const* rev = "$Revision: 1.12 $$Date: 2016/11/04 13:57:24 $";

// This programm attemps to do a bunch of random mallocs and frees.
// 
// make malloc_free CXX="g++ -std=c++0x"
// CXX="g++ -std=c++0x" make -e malloc_free   # NOTE "-e"

#include <stdio.h>      // printf
#include <getopt.h>             // getopt_long, {no,required,optional}_argument, extern char *optarg; extern int opt{ind,err,opt}
#include <stdlib.h>     // malloc, rand - range [0, RAND_MAX]  period at least 2**32
// random: 0 to RAND_MAX -- period 16 * ((2^31) - 1), has initstate() and setstate() functions
#include <libgen.h>             // basename
#include <time.h>               // time()
#include <sys/time.h>           // gettimeofday
#include <unistd.h>             // getpid()
#include <stdint.h>             // uint64_t
#include <string.h>             // memset
#include <vector>
#include <tuple>                // pair
#include <set>
#include <mutex>
#include <boost/thread.hpp>
#include "send_gmetric.h"

#define TRACE_NAME "malloc_free"
#if 1
# include "trace.h"             // TRACE
#else
# define TRACE( vv, ... ) do { if (opt_v>=vv) printf( __VA_ARGS__ ); } while (0)
#endif

#define USAGE "\
  usage: %s [-hv?]\n\
\n\
example: %s\n\
\n\
option:\n\
  -V             print version and exit\n\
 --help\n\
 --seed=         random number generator seed\n\
 --min=          min alloc  default=%u\n\
 --max=          max alloc  default=%u\n\
 -t --time=      seconds\n\
 --new           use new/delete\n\
 --no-seed       \n\
 --ave=          default=%d\n\
 --threads=      default=%u\n\
", basename(argv[0]), basename(argv[0]), opt_min_alloc, opt_max_alloc\
        , opt_ave, opt_threads

/* GLOBALS */
uint64_t opt_hi_water = 1 * (1024 * 1024 * 1024);
uint64_t opt_lo_water = 8 * (1024 * 1024);
unsigned opt_seed = 2, opt_seed_ = 1; // 2 variables -- to allow any possible seed value
unsigned opt_max_alloc = 128 * (1024);
unsigned opt_min_alloc = 1 * (1024);
int opt_v = 0;
int opt_new = 0;
unsigned opt_ave = 100000;
int opt_threads = 1;
int opt_time = 25;
int g_stop = 0;

uint64_t human(char* optarg)
{
    char* endptr;
    uint64_t ret = strtoul(optarg, &endptr, 0);
    if (*endptr == 'k' || *endptr == 'K') ret *= 1024;
    else if (*endptr == 'm' || *endptr == 'M') ret *= 1024 * 1024;
    else if (*endptr == 'g' || *endptr == 'G') ret *= 1024 * 1024 * 1024;
    return ret;
}

void parse_args(int argc, char* argv[])
{
    // parse opt, optargs, and args
    while (1)
    {
        int opt;
        static struct option long_options[] = {
            // name            has_arg          *flag  val
            {"help", no_argument, 0, 'h'},
            {"seed", required_argument,0, 's'},
            {"hi", required_argument,0, 1},
            {"lo", required_argument,0, 2},
            {"min", required_argument,0, 3},
            {"max", required_argument,0, 4},
            {"no-seed", no_argument, 0, 5},
            {"ave", required_argument,0, 6},
            {"threads", required_argument,0, 7},
            {"time", required_argument,0, 't'},
            {0, 0, 0, 0}
        };
        opt = getopt_long(argc, argv, "?hvVs:t:",
            long_options, NULL);
        if (opt == -1) break;
        switch (opt)
        {
        case '?': case 'h': printf(USAGE);
            exit(0);
            break;
        case 'V': printf("%s\n", rev);
            exit(0);
            break;
        case 'v': ++opt_v;
            break;
        case 's': opt_seed = strtoul(optarg, NULL, 0);
            opt_seed_ = 1;
            break;
        case 't': opt_time = strtoul(optarg, NULL, 0);
            break;
        case 1: opt_hi_water = human(optarg);
            break;
        case 2: opt_lo_water = human(optarg);
            break;
        case 3: opt_min_alloc = human(optarg);
            break;
        case 4: opt_max_alloc = human(optarg);
            break;
        case 5: opt_seed_ = 0;
            break;
        case 6: opt_ave = strtoul(optarg, NULL, 0);
            break;
        case 7: opt_threads = strtoul(optarg, NULL, 0);
            break;
        default:
            printf("?? getopt returned character code 0%o ??\n", opt);
            exit(1);
        }
    }
}

uint64_t gettimeofday_us(void)
{
    struct timeval tv;
    gettimeofday(&tv, NULL);
    return (uint64_t)tv.tv_sec * 1000000 + tv.tv_usec;
}

struct Stats
{
    Stats() : count_(0)
        , running_tot_us_(0)
        , running_tot_MB_(0.0) {}

    void update(double MB, int64_t delta_us)
    {
        mtx_.lock();
        running_tot_MB_ += MB;
        running_tot_us_ += delta_us;
        times_per_.insert((double)delta_us / MB);
        ++count_;
        mtx_.unlock();
    }

    void get_clear(double& tot_MB, int64_t& tot_delta_us, int& count, double& min, double& max)
    {
        mtx_.lock();
        tot_MB = running_tot_MB_;
        tot_delta_us = running_tot_us_;
        count = count_;
        if (count == 0)
            min = max = 0;
        else
        {
            min = *(times_per_.begin());
            max = *(times_per_.rbegin());
        }
        running_tot_MB_ = 0.0;
        running_tot_us_ = 0;
        times_per_.clear();
        count_ = 0;
        mtx_.unlock();
    }

private:
    int count_;
    int64_t running_tot_us_;
    double running_tot_MB_;
    std::multiset<double> times_per_;
    std::mutex mtx_;
};

struct Malloc
{
    Malloc(Stats& stats) : num_mallocs(0)
        , stats_(stats) {}

#define NUM_DELTAS opt_ave
    int num_mallocs;
    Stats& stats_;

    // alloc_bytes is an output -- This function returns a ptr to the memory
    // and the num bytes allocated.
    void* malloc_opt(unsigned* alloc_bytes)
    {
        void* ptr;
        if ((opt_max_alloc - opt_min_alloc) == 0)
            *alloc_bytes = opt_max_alloc;
        else
            *alloc_bytes = (random() % (opt_max_alloc - opt_min_alloc)) + opt_min_alloc;

        uint64_t t0 = gettimeofday_us();

        ptr = malloc(*alloc_bytes);
        if (ptr == NULL)
        {
            perror("malloc");
            exit(1);
        }
        unsigned ii;
        for (ii = 0; ii < *alloc_bytes; ii += 4096)
            *(int*)((char*)ptr + ii) = 1;
        ii /= 4096; // convert offset to pages

        int64_t delta_us = gettimeofday_us() - t0;
#       define PAGES_PER_MB (0x100000/4096.0)
        double MB = (double)ii / PAGES_PER_MB;
        TLOG(TLVL_INFO) << "malloc " << *alloc_bytes << " bytes ii=" << ii << " MB=" << MB;

        stats_.update(MB, delta_us);

        return ptr;
    }
};


static void do_work(Stats* stats)
{
    std::vector<std::pair<void*, int>> ptr_vec;
    uint64_t total_allocation = 0;
    int direction; // 1 is up, 0 is down.
    enum
    {
        up,
        down
    };

    Malloc mm(*stats);

    while (total_allocation < opt_lo_water)
    {
        unsigned alloc_bytes;
        int8_t* ptr = (int8_t*)mm.malloc_opt(&alloc_bytes);
        std::pair<void*, int> xx(ptr, alloc_bytes);
        ptr_vec.push_back(xx);
        total_allocation += alloc_bytes;
    }
    direction = up;

    printf("ptr_vec.size()=%ld\n", ptr_vec.size());
    while (!g_stop)
    {
        long rr = random();
        // for the "up" direction, higher probability for malloc
        long tst = (direction == up) ? RAND_MAX / 4 * 3 : RAND_MAX / 4 * 1;
        int malloc1_or_free0 = rr < tst;

        if (malloc1_or_free0)
        {
            // do mallocs
            unsigned alloc_bytes;
            int8_t* ptr = (int8_t*)mm.malloc_opt(&alloc_bytes);
            std::pair<void*, int> xx(ptr, alloc_bytes);
            ptr_vec.push_back(xx);
            total_allocation += alloc_bytes;
            if (total_allocation > opt_hi_water && direction == up)
            {
                TLOG(TLVL_WARNING) << "direction=down";
                direction = down;
            }
        }
        else
        {
            // do frees
            if (ptr_vec.size() == 0)
            {
                printf("size()==0 - continue\n");
                continue;
            }
            unsigned free_idx = random() % ptr_vec.size();
            std::pair<void*, int> xx(ptr_vec[free_idx]);
            ptr_vec.erase(ptr_vec.begin() + free_idx);
            TLOG(TLVL_INFO) << "free " << xx.second;
            free(xx.first);
            total_allocation -= xx.second;
            if (total_allocation < opt_lo_water && direction == down)
            {
                TLOG(TLVL_WARNING) << "direction=up";
                direction = up;
            }
        }
    }
}

void send_metrics(double ave, double min, double max, double MBperalloc, int32_t threads)
{
#       define CLUSTER "mu2e DAQ"
#       define GROUP   "memory"
    send_gmetric("ave", std::to_string(ave).c_str(), "double", "us/MB"
        , "both", 30, 0, GROUP, CLUSTER, "ave us/MB", "ave us/MB");
    send_gmetric("min", std::to_string(min).c_str(), "double", "us/MB"
        , "both", 30, 0, GROUP, CLUSTER, "min us/MB", "min us/MB");
    send_gmetric("max", std::to_string(max).c_str(), "double", "us/MB"
        , "both", 30, 0, GROUP, CLUSTER, "max us/MB", "max us/MB");
    send_gmetric("ave_MBperalloc", std::to_string(MBperalloc).c_str(), "double", "MB/alloc"
        , "both", 30, 0, GROUP, CLUSTER, "ave MB/alloc", "ave MB/alloc");
    send_gmetric("threads", std::to_string(threads).c_str(), "int32", "threads"
        , "both", 30, 0, GROUP, CLUSTER, "threads mallocing", "threads mallocing");
}


int main(int argc, char* argv[])
{
    parse_args(argc, argv);

    if (opt_hi_water < opt_lo_water)
    {
        printf("changeing hi_water from %lu to %lu\n", opt_hi_water, opt_lo_water);
        opt_hi_water = opt_lo_water;
    }
    if (opt_max_alloc < opt_min_alloc)
    {
        printf("changeing max_alloc from %u to %u\n", opt_max_alloc, opt_min_alloc);
        opt_max_alloc = opt_min_alloc;
    }

    if (!opt_seed_)
        opt_seed = time(NULL) * getpid();
    srandom(opt_seed);
    printf("opt_seed=%u RAND_MAX=%d random=%ld\n", opt_seed, RAND_MAX, random());
    printf("lo=%ld hi=%ld min=%u max=%u\n", opt_lo_water, opt_hi_water, opt_min_alloc, opt_max_alloc);

    init_gmetric("/etc/ganglia/gmond.conf");


    std::vector<boost::thread> threads(opt_threads);
    std::vector<Stats> stats(opt_threads);
    for (auto nn = 0; nn < opt_threads; ++nn)
    {
        boost::thread::attributes attrs;
        attrs.set_stack_size(4096 * 2000); // 8000 KB
        try {
            threads[nn] = boost::thread(attrs, boost::bind(&do_work, &stats[nn]));
        }
        catch (const boost::exception& e)
        {
            TLOG(TLVL_ERROR) << "Caught boost::exception starting malloc_free thread: " << boost::diagnostic_information(e) << ", errno=" << errno;
            std::cerr << "Caught boost::exception starting malloc_free thread: " << boost::diagnostic_information(e) << ", errno=" << errno << std::endl;
            exit(5);
        }
    }

    uint64_t start_us = gettimeofday_us();
    uint64_t end_us = start_us + (1000000L * opt_time);
    usleep(5000000);
    for (auto cc = 2; 1; ++cc)
    {
        double tot_ave_usperMB = 0.0, tot_min_usperMB = 0.0, tot_max_usperMB = 0.0, tot_MBperalloc = 0.0;
        for (auto nn = 0; nn < opt_threads; ++nn)
        {
            double tot_MB;
            int64_t tot_us;
            int count;
            double min, max;
            stats[nn].get_clear(tot_MB, tot_us, count, min, max);
            tot_ave_usperMB += (tot_MB != 0.0) ? (double)tot_us / tot_MB : 0.0;
            tot_min_usperMB += min;
            tot_max_usperMB += max;
            tot_MBperalloc += count ? tot_MB / count : 0;
            TLOG(6) << "nn=" << nn << " tot_MB=" << std::setprecision(4) << tot_MB << " count=" << count;
        }
        if ((cc % 5) == 0)
            TLOG(TLVL_ERROR) << opt_threads
            << " ave malloc usec/MB ave=" << std::setprecision(4) << tot_ave_usperMB / opt_threads
            << " min=" << tot_min_usperMB / opt_threads << " max=" << tot_max_usperMB / opt_threads
            << " MB/alloc=" << std::setprecision(1) << tot_MBperalloc / opt_threads;

        send_metrics(tot_ave_usperMB / opt_threads
            , tot_min_usperMB / opt_threads
            , tot_max_usperMB / opt_threads
            , tot_MBperalloc / opt_threads
            , opt_threads);

        uint64_t target = start_us + (5000000L * cc);
        uint64_t now_us = gettimeofday_us();
        int64_t slp = target - now_us;
        if (slp > 0)
        {
            if ((now_us + slp) > end_us)
                break;
            usleep(slp);
        }
        else
        {
            printf("cc=%d slp=%ld\n", cc, slp);
            if (now_us > end_us)
                break;
        }
    }
    send_metrics(0.0, 0.0, 0.0, 0.0, 0);
    g_stop = 1;
    for (auto nn = 0; nn < opt_threads; ++nn)
    {
        threads[nn].join();
    }
    printf("malloc_free finished.\n");
    return (0);
} // main