Beauty analyzer

beauty.hh

void Beauty_VIS(const char* Xbuf, unsigned w, unsigned h);
void Beauty_INPUT();

beauty.cc

#include <cmath>
#include <cstring>
#include <algorithm>

#include <vector>
#include <map>

#include <complex>

/*
#include "../gfx.h"
#include "../movie.h"
#include "../screenshot.h"
#include "../snapshot.h"
#include "../ppu.h"
*/

#include "beauty.hh"
#include "coroutine.h"

#include "System.h"
#include "movie.h"
#include "gb/gbGlobals.h"
#include "GBA.h"
#include "Globals.h"
extern struct EmulatedSystem emulator;
#define theEmulator (emulator)

typedef std::complex<double> complex;

int do_beauty_analysis = 0;

/* Analyzes beauty in the given GB/GBA screenshot.
 */

static const unsigned char* xbuf = 0;
static unsigned width;

static int skippattern(int p)
{
    return 1 + ((p/7)%3);
}
/* Calculate average tint in given section of the screen. */
static complex GetAverageTint(int x0,int y0, int x1,int y1)
{
    double avg_i = 0, avg_q = 0, avg_y = 0;
    double count = 0;

    for(int y=y0; y<=y1; y+=skippattern(x0+y))
        for(int x=x0; x<=x1; x+=skippattern(y+x))
        {
            unsigned rr = xbuf[(x + y * width)*4+0];
            unsigned gg = xbuf[(x + y * width)*4+1];
            unsigned bb = xbuf[(x + y * width)*4+2];
            
            double r = rr / 255.0;
            double g = gg / 255.0;
            double b = bb / 255.0;
            
            double y = 0.299  * r + 0.587  * g +  0.114  * b;
            double i = 0.5957 * r - 0.2744 * g -  0.3212 * b;
            double q = 0.2114 * r - 0.5226 * g +  0.3111 * b;
            
            double pow = 1;
            
            avg_i += i*y;
            avg_q += q*y;
            avg_y += y;
            
            count += pow;
        }

    avg_i /= count;
    avg_q /= count;
    avg_y /= count;
    
    complex res(avg_i, avg_q); res *= 100;
    
    double y = std::abs(res);
    if(y >= 0.0) res *= std::log(1 + y) / (1 + y);
    return res;
    
    //return complex ( atan2(avg_i, avg_q) , std::log(1 + avg_y));
}

struct region
{
    int x0,x1, y0,y1;
    
    complex tint;
    
    int sprite_jitter;
};
struct sprite
{
    int x,y, width,height;
};
static bool SOBJcompare(const sprite& a, const sprite& b)
{
    return std::memcmp(&a, &b, sizeof(a)) < 0;
}

static void Do_GB_Sprites(sprite sprites[128])
{
  int count = 0;
  int size = (register_LCDC & 4);

  if((register_LCDC & 0x80))
  {
      if((register_LCDC & 2) && (layerSettings & 0x1000)) {
        int yc = register_LY;
        int address = 0xfe00;
        int count = 0;
        for(int i = 0; i < 40; i++) {
          int y = gbMemory[address++];
          int x = gbMemory[address++];
          int tile = gbMemory[address++];
          if(size) tile &= 254;
          int flags = gbMemory[address++];

          if(x > 0 && y > 0 && x < 168 && y < 160)
          {
            if(size) {
              sprites[count].x = x-8;
              sprites[count].y = y;
              sprites[count].width = 8;
              sprites[count].height = 16;
              count++;   
              if(count >= 128) return;
            } else {
              sprites[count].x = x-8;
              sprites[count].y = y;
              sprites[count].width = 8;
              sprites[count].height = 8;
              count++; 
              if(count >= 128) return;
            }
          }
        }   
      }
  }
  while(count < 128) sprites[count++].y = 256;
}

static double CalculateBeauty(unsigned w,unsigned h)
{
    std::vector<region> regions;
    regions.reserve(4*3);
    for(unsigned x=0; x<4; ++x)
        for(unsigned y=0; y<3; ++y)
        {
            region tmp;
            tmp.tint=0;
            tmp.sprite_jitter=0;
            tmp.x0 = x*w/4;
            tmp.x1 = (x+1)*w/4;
            tmp.y0 = y*h/3;
            tmp.y1 = (y+1)*h/3;
            regions.push_back(tmp);
        }
/*    
     regions[5] =
    {
        // corners: nw, ne, sw, se, middle
        { w*0 /99,w*50/99, h*0 /99,h*50/99, 0, 0 },
        { w*50/99,w*99/99, h*0 /99,h*50/99, 0, 0 },
        { w*0 /99,w*50/99, h*50/99,h*99/99, 0, 0 },
        { w*50/99,w*99/99, h*50/99,h*99/99, 0, 0 },
        { w*33/99,w*55/99, h*33/99,h*55/99, 0, 0 }
    };
*/

    static sprite old_sprites[128];
    sprite new_sprites[128];
    
    if(true)
    {
        // init new_sprites
        Do_GB_Sprites(new_sprites);
    }
    
    std::sort(new_sprites, new_sprites+128, SOBJcompare);
    
    for(unsigned b=0; b<128; ++b)
    {
        sprite& spr = new_sprites[b];
        sprite& old = old_sprites[b];
        
        // If the sprite appears to have moved, compensate for scrolling and try again
        if(spr.x != old.x || spr.y != old.y)
        {
            //spr.x -= ScrollPointers[spr.y];
            //spr.y -= VScroll;
        }
        
        if(std::memcmp(&spr, &old, sizeof(spr)) != 0)
        {
            old = spr;
            
            int x = spr.x, y = spr.y;
            int width = spr.width, height = spr.height;
            
            if(y >= 239) continue;
            
            for(unsigned a=0; a< regions.size(); ++a)
            {
                // Check if this sprite affects this region
                region& reg = regions[a];
                
                if(x+width  >= reg.x0 && x < reg.x1
                && y+height >= reg.y0 && y < reg.y1)
                {
                    reg.sprite_jitter += 1;
                    if(spr.x != old.x || spr.y != old.y)
                        reg.sprite_jitter += 2;
                }
            }
        }
    }

    double totaltint = 0;
    for(unsigned a=0; a< regions.size(); ++a)
    {
        region& reg = regions[a];
        reg.tint = GetAverageTint(reg.x0, reg.y0, reg.x1, reg.y1);
        totaltint += std::abs(reg.tint);
    }
    totaltint /= regions.size();
    
    /*
    PLAN
    
    Analyze five sections of this screen.
    
    - Each four quarters of the screen
    - And the middle (slightly larger than a quarter?)
    
    Analyze:
    - Average tint
    - Amount of motion (ignore global motion)
    
    Value:
    - Non-bleak tints
    - Different tint in each section of screen
    - Motion appears in at least two different sections of screen
    
    */
    
    if(do_beauty_analysis >= 3)
    {
        fprintf(stderr, "analysis: ");
        for(unsigned a=0; a< regions.size(); ++a)
        {
            region& reg = regions[a];
            fprintf(stderr, "(%3.1f<%+3.1f)(%2d) ",
                std::abs(reg.tint),
                std::arg(reg.tint),
                reg.sprite_jitter);
        }
        fprintf(stderr, "\n");
    }

/*
    complex t = regions[0].tint + regions[1].tint + regions[2].tint + regions[3].tint;
    t /= 4;
*/

    double interestingness = 0;
    
    if(totaltint >= 0.1)
    {
        for(unsigned a=0; a<regions.size(); ++a)
            for(unsigned b=a+1; b<regions.size(); ++b)
            {
                double mul = 1;
                if(regions[a].x0 != regions[b].x0) mul *= 2;
                if(regions[a].y0 != regions[b].y0) mul *= 2;

                double tintdiff = std::abs(regions[a].tint - regions[b].tint) * 0.001;
                interestingness += tintdiff * mul;
                
                interestingness += std::log(2 + regions[a].sprite_jitter / 4.0)
                                 * std::log(2 + regions[b].sprite_jitter / 4.0)
                                 * mul * 100;
            }
    }
/*
        std::abs(regions[0].tint - regions[1].tint)
      + std::abs(regions[0].tint - regions[2].tint)
      + std::abs(regions[0].tint - regions[3].tint)
      + std::abs(regions[1].tint - regions[2].tint)
      + std::abs(regions[1].tint - regions[3].tint)
      + std::abs(regions[2].tint - regions[3].tint)
      + std::abs(regions[4].tint - t)
      +  (std::log(1 + regions[0].sprite_jitter / 8.0)
        * std::log(1 + regions[3].sprite_jitter / 8.0)
      +   std::log(1 + regions[1].sprite_jitter / 8.0)
        * std::log(1 + regions[2].sprite_jitter / 8.0)
      +   std::log(1 + regions[3].sprite_jitter / 4.0)
         );
*/
    return interestingness;
}

struct SaveState
{
    std::vector<unsigned char> Data;
    
    static const unsigned StateSize = 400000;

    void Create()
    {
        extern long LastMemStateLength;
        Data.resize(StateSize);
        
        int l = theEmulator.emuWriteMemState((char*)&Data[0], Data.size());
        
        if(l) Data.resize(LastMemStateLength);
        
        fprintf(stderr, "Freeze (ret %d) : %ld bytes\n", l, LastMemStateLength); fflush(stderr);
    }
    void Load()
    {
        int l = theEmulator.emuReadMemState((char*)&Data[0], Data.size());
        fprintf(stderr, "Unfreeze (ret %d): %u bytes\n", l, Data.size()); fflush(stderr);
    }
};

static std::vector<double> interestingness_map;
static std::map<int, SaveState> some_savestates;
static std::vector<unsigned> review_results;

static bool LocateBeauty(unsigned frameno)
{
    scrBegin;
    if(true) /* scope */
    {
        std::map<int, SaveState>::iterator
            i = some_savestates.lower_bound(frameno);

        for(;;)
        {
            if(i != some_savestates.end())
            {
                if(i->first <= frameno)
                {
                    unsigned framecount = VBAMovieGetFrameCounter();

                    if(framecount > frameno || i->first > framecount)
                    {
                        fprintf(stderr, "Aiming for frame %u, loading one that gives %u hopefully\n",
                            frameno, i->first);
                        i->second.Load();
                    }
                    else
                    {
                        fprintf(stderr, "Aiming for frame %u. Playing from %d\n",
                            frameno, framecount);
                    }
                    
                    fflush(stderr);
                    break;
                }
            }
            if(i == some_savestates.begin()) break;
            --i;
        }
    }
    
    while(VBAMovieGetFrameCounter() < frameno)
    {
      //  fprintf(stderr, "Frame %u\n", VBAMovieGetFrameCounter());
      //  fflush(stderr);
      //  extern bool pauseNextFrame;
      //  pauseNextFrame = true;
        scrReturn(false);
    }
    scrFinish(true);
}

extern "C" void CloseStuff(int signum);

static void CaptureBeauty()
{
    scrBegin;
    static unsigned a;
    for(a = 0; a < review_results.size(); ++a)
    {
        fprintf(stderr, "Capturing beauty %u at frame %u\n", a, review_results[a]);
        fflush(stderr);
        
        while(!LocateBeauty(review_results[a]))
        {
            scrReturnV;
        }

        /* capture it */

        char beauty_fn[64];
        sprintf(beauty_fn, "beauty_%u.png", a);
        fprintf(stderr, "- Capturing %s, have frame %u now\n", beauty_fn,
            VBAMovieGetFrameCounter());
        fflush(stderr);
        
        theEmulator.emuWritePNG(beauty_fn);
    }
    
    /* all beauties have been analyzed, end stuff now */
    fprintf(stderr, "Done with all beauties\n");
    
    exit(0);
    
    scrFinishV;
}

struct section
  { unsigned sect; double value;
    bool operator< (const section& sect) const
    { return value > sect.value; }
    bool operator== (const section& sect) const
    { return value == sect.value; }
  };


static void ReviewBeauty()
{
    const unsigned framecount = VBAMovieGetFrameCounter();
    const unsigned num_sections = 20;std::max(
      std::min(50, (int)(framecount / 60)),
        // not more than 50
        // not more than one per each second of movie
      (int)(framecount / 600)
        // but still at least one per each 10 seconds of movie
    );
    const unsigned num_results  = 20;std::min(
      std::min(20, (int)( framecount / (3600/5) ) ),
         // not more than 15
         // not more than 5 per each minute of movie
      (int)(num_sections));
    
    std::vector<section> sects(num_sections);
    
    for(unsigned a=0; a<num_sections; ++a)
        sects[a].sect = a;
    
    for(unsigned a=0; a<interestingness_map.size(); ++a)
    {
        const double val = interestingness_map[a];
        const unsigned sect = a * num_sections / interestingness_map.size();
        if(val > sects[sect].value) sects[sect].value = val;
    }
    
    std::sort(sects.begin(), sects.end());
    
    for(unsigned a=0; a<num_results; ++a)
    {
        unsigned sect = sects[a].sect;
        unsigned sect_begin = (sect) * interestingness_map.size() / num_sections;
        unsigned sect_end = (sect+1) * interestingness_map.size() / num_sections;
        
        for(unsigned b=sect_begin; b<sect_end; ++b)
            if(interestingness_map[b] == sects[a].value)
            {
                fprintf(stderr, "Selected beauty at frame %u (%g)\n", b, sects[a].value);
                fflush(stderr);
                review_results.push_back(b);
                break;
            }
    }
    std::sort(review_results.begin(), review_results.end());
}

static bool finished_recording = false;

void Beauty_VIS(const char* Xbuf, unsigned w, unsigned h)
{
    //fprintf(stderr, "vis %d\n", do_beauty_analysis);
    
    if(!do_beauty_analysis) return;
    
    xbuf = (const unsigned char*) Xbuf;
    width = w;
    
    if(!finished_recording) /* Record interestingness */
    {
        double interestingness = CalculateBeauty(w,h);
        const unsigned framecount = VBAMovieGetFrameCounter();

        if(do_beauty_analysis >= 2)
        {
            std::printf(
                "Beauty %u: %g\n", framecount, interestingness);
            std::fflush(stdout);
        }

        if(interestingness_map.size() <= framecount)
            interestingness_map.resize(framecount+1);
        interestingness_map[framecount] = interestingness;
        
        static unsigned maxframe = ~0;
        if(maxframe == (unsigned)~0)
        {
            const char* env = getenv("MAXFRAMES");
            maxframe = env ? atoi(env) : 0;
        }

        finished_recording = framecount == maxframe-2;
        
        if(!finished_recording)
        {
            return;
        }
    }
}

void Beauty_INPUT()
{
    //fprintf(stderr, "in %d\n", do_beauty_analysis);
    
    if(!do_beauty_analysis) return;
    
    if(!finished_recording) /* Record interestingness */
    {
        static unsigned lastcount=0;
        const unsigned framecount = VBAMovieGetFrameCounter();
        if(framecount != 0 && (framecount == 1 || !(framecount % 100))
        && framecount != lastcount)
        {
            fprintf(stderr, "Creating savestate @ %u\n", framecount);
            fflush(stderr);
            some_savestates[framecount].Create();
            lastcount = framecount;
        }
        return;
    }
    
    /* Analyze interestingness */
    if(review_results.empty())
        ReviewBeauty();

    CaptureBeauty();
}


void BeautyInit() __attribute__ ((constructor));
void BeautyInit()
{
    char* env = getenv("BEAUTY");
    do_beauty_analysis = env ? atoi(env) : 0;
}