here is an example, demo video will follow later
It loads audio and image
anayzes audio via fft
converts fft to ERB scale (5 bands here)
maps this on 255 values 0..1
these values are then used as lookup for brightness (greyscale) of pixels
in the image
here the image is converted to greyscale, and added on the color version
with "overlay" blending mode (or "hard light", "multiply", "screen" etc)
the brightness in the greyscale is used to determin the blend amount
does not run realtime
Video will follow, maybe someone has more ideas or improvements
Code: Select all
set_pixel_size( WINDOW_XSIZE / 480 )
resize( get_screen(), WINDOW_XSIZE, WINDOW_YSIZE )
//Input Image
img = load( "images/test_5b.jpg" )
//Input WAV file:
filename = "audio.wav"
//Output MJPEG AVI video file:
avi_filename = "test 2.avi"
//Graphics options:
showfps = 1;
xsize = 1024
ysize = 600
if avi_filename != 0
{
//xsize = 1280
//ysize = 720
}
fps = 25
//_---------------------------------
if xsize == 0 { ss = 600 } else { ss = xsize }
set_pixel_size( WINDOW_XSIZE / ss )
resize( get_screen(), WINDOW_XSIZE, WINDOW_YSIZE )
scr = get_screen()
if xsize != 0 && ysize != 0 { resize( scr, xsize, ysize ) }
xsize = get_xsize( scr )
ysize = get_ysize( scr )
hxsize = xsize / 2
hysize = ysize / 2
if fps == 0 { fps = 25 }
wav = load( filename )
wav_ptr = 0
wav_size = get_size( wav ) //number of frames
wav_channels = wav.channels
wav_amp_max = 256
samplerate = wav.sample_rate
sample_rate_scale = 1
type = get_type( wav )
if type == INT16 { wav_amp_max = 1 << 15 }
if type == INT32 { wav_amp_max = 1 << 30 }
if type == FLOAT32 { wav_amp_max = 1 }
include "../../../lib/mjpeg.pixi"
if avi_filename != 0
{
vo = 1
vo_f = fopen( avi_filename, "wb" )
if vo_f <= 0 { logf( "Can't open video file for writing\n" ) halt }
vo_encoder = mjpeg_encoder_open(
fps,
xsize,
ysize,
90, //Quality
wav_channels, //Audio channels
wav.sample_rate * sample_rate_scale, //Audio frames per second
get_type( wav ), //Audio sample type
MJPEG_ENCODER_FLAG_USEINDEX | MJPEG_ENCODER_FLAG_HASSOUND, //Flags
vo_f )
vo_audio_buf_size = mjpeg_encoder_get_audio_size( vo_encoder ) //Number of frames per audio chunk
vo_audio_buf = new( vo_audio_buf_size * wav_channels, 1, get_type( wav ) )
vo_audio_ch_bufs = new( wav_channels, 1, INT )
i = 0 while i < wav_channels { vo_audio_ch_bufs[ i ] = new( vo_audio_buf_size, 1, get_type( wav ) ) i + 1 }
logf( "Audio buffer size: %d frames\n", vo_audio_buf_size )
}
else
{
set_audio_callback( audio_callback, 0, wav.sample_rate * sample_rate_scale, get_type( wav ), wav_channels, AUDIO_FLAG_INTERP2 )
rate1 = get_audio_sample_rate( 0 )
rate2 = get_audio_sample_rate( 1 )
logf( "Local (defined by the set_audio_callback()) sample rate: %d Hz\n", rate1 )
logf( "Global (defined in the global Pixilang preferences) sample rate: %d Hz\n", rate2 )
if rate1 != rate2
{
logf( "%d != %d, so resampling will be enabled\n", rate1, rate2 )
}
}
//-------------------
gfx_init()
start_timer( 0 )
while( 1 )
{
transp( 256 )
clear();
// analyse audio ######################################
// FFT
clean(ftbufi)
clean(ftbufr)
$p = wav_ptr
$t = $p / wav_size
$i = 0 while $i < ftsize
{
$v = wav[ $p + $i * wav_channels ] / wav_amp_max
vals[ $i ] = $v
$winlen = 2* wav.sample_rate/fps
// half sine window
//if $i <= $winlen{$win = sin(M_PI*$i/$winlen)} else{$win =0 }
//or cosine window
if $i <= $winlen{$win = cos(2*M_PI*$i/$winlen)*0.5 + 0.5} else{$win =0 }
ftbufi[ $i ] = 0
ftbufr[ $i ] = $v*$win
$i + 1
}
fft(1, ftbufi,ftbufr,ftsize)
// get magnitude
$i = 0 while $i < hftsize
{
ftmag[ $i ] = sqrt(ftbufr[$i]*ftbufr[$i]+ftbufi[$i]*ftbufi[$i])
if ftmag[ $i ] >= ftmags[ $i ] {
ftmagsmoo[ $i ] = ftmag[$i]
}else{ ftmagsmoo[ $i ] = ftmagsmoo[ $i ] * 0.92 }
$i + 1
}
// map on ERB scale
$size = erbsize
$ERBmin = 1
$ERBrange = 38
//clean(erbbuf)
$i = 0 while $i < $size
{
//$warpf = i*wav.sample_rate/ftsize
//$warperb = 21.4 * log10(1 + 0.00437 * $warpf)
//$warpi1 = ($warperb *xsize/60 +0.5)
//map erb range-min, min erb:
$erb = $ERBrange*$i/$size + $ERBmin
$erbf = ($erb) * ( 1/21.4)
$erbf = pow(10,$erbf)
$erbf = $erbf - 1
$erbf = $erbf * (1/ 0.00437)
$erbift1 = $erbf/(wav.sample_rate/ftsize)
if $erbift1 < 0 {$erbift1 =0}
$erbf = ($erb + $ERBrange/$size) * ( 1/21.4)
$erbf = pow(10,$erbf)
$erbf = $erbf - 1
$erbf = $erbf * (1/ 0.00437)
$erbift2 = $erbf/(wav.sample_rate/ftsize)
if $erbift2 < 0 {$erbift2 = 0}
erbbuf[$i]=0
$j = 0 while $j < $erbift2 - $erbift1
{
erbbuf[$i] = erbbuf[$i] + ftmagsmoo[$erbift1 +$j]
$j = $j +1
}
// convert to dB
erbbuf[$i] = atodb(erbbuf[$i])
//get average
if $i == 0 { $dbav = erbbuf[$i] }else{ $dbav = $dbav + erbbuf[$i] }
$i +1
}
$dbav = $dbav / $size
//------------------------------------------
// create light correction map
$size = 256
//$dbmult = 2
//$dblift = 6
$i = 0 while $i < $size
{
$erbi = ($i * erbsize / $size )
$frac = mod($erbi,1)
$erbi = $erbi div 1
$v = dbtoa(erbbuf[$erbi])
$v2 = dbtoa(erbbuf[$erbi +1])
$v = $v + ($v2 - $v) * $frac
//lightbuf[$i] = ($v- $dbav)/$dbmult
lightbuf[$i] = $v * 32//( $size / erbsize)
if lightbuf[$i] > 1 { lightbuf[$i] = 1}
if lightbuf[$i] < 0 { lightbuf[$i] = 0}
if lightbuf[$i] > lightbufsmoo[$i] {
lightbufsmoo[$i] = lightbuf[$i]
}else{ lightbufsmoo[$i] = lightbufsmoo[$i]*0.92}
$i + 1
}
// GRAFIC
//convert img2c to grey and relight
$offsetmax = 0 // offset not used here
$i = 0 while $i < get_size(img2b){
// get greyscale
$gr = get_red(img2c[$i]) + get_green(img2c[$i]) + get_blue(img2c[$i])
$gr = $gr div 3
$cgr = get_color( $gr,$gr,$gr)
$idxnew = ( $i + lightbufsmoo[$gr] * $offsetmax ) % get_size(img2a)
// hard light with with greyscale:
/*
if $gr < 127{
$gr_m = 2 * $gr
$pixelr = get_red(img2a[$idxnew])*$gr_m div 255
$pixelg = get_green(img2a[$idxnew])*$gr_m div 255
$pixelb = get_blue(img2a[$idxnew])*$gr_m div 255
} else {
$gr_m = 255 - $gr
$pixelr = 255 - 2*(255-get_red(img2a[$idxnew]))*$gr_m div 255
$pixelg = 255 - 2*(255-get_green(img2a[$idxnew]))*$gr_m div 255
$pixelb = 255 - 2*(255-get_blue(img2a[$idxnew]))*$gr_m div 255
}
$pixelrgb = get_color($pixelr,$pixelg,$pixelb)
*/
// overlay light with with greyscale:
if $gr < 127{
$gr_m = $gr
$pixelr = 2*get_red(img2a[$idxnew])*$gr_m div 255
$pixelg = 2*get_green(img2a[$idxnew])*$gr_m div 255
$pixelb = 2*get_blue(img2a[$idxnew])*$gr_m div 255
} else {
$gr_m = 255 - $gr
$pixelr = 255 - 2*(255-get_red(img2a[$idxnew]))*$gr_m div 255
$pixelg = 255 - 2*(255-get_green(img2a[$idxnew]))*$gr_m div 255
$pixelb = 255 - 2*(255-get_blue(img2a[$idxnew]))*$gr_m div 255
}
$pixelrgb = get_color($pixelr,$pixelg,$pixelb)
// screen light with greyscale
/*
$gr_m = 255 - $gr
$pixelr = 255 - (255-get_red(img2a[$idxnew]))*$gr_m div 255 % 255
$pixelg = 255 - (255-get_green(img2a[$idxnew]))*$gr_m div 255 % 255
$pixelb = 255 - (255-get_blue(img2a[$idxnew]))*$gr_m div 255 % 255
$pixelrgb = get_color($pixelr,$pixelg,$pixelb)
*/
// mix hard light according to ERB scale spectrum
img2b[$idxnew] = get_blend(img2a[$i],$pixelrgb,255*lightbufsmoo[$gr])
$i +1
}
//pixi( img2b, 0, 0)
pixi( img2b, 0, 0,WHITE,2,2 )
// FRAME########################################################
if showfps {
ts = ""
sprintf( ts, "FPS:%u", FPS )
print( ts, -get_xsize( get_screen() ) / 2 + 16, -get_ysize( get_screen() ) / 2 + 16, WHITE, TOP | LEFT )
}
if vo
{
//Video export:
audio_callback( 0, 0, vo_audio_ch_bufs, vo_audio_buf_size, 0, -1, 0 )
i = 0 while i < wav_channels
{
copy( vo_audio_buf, vo_audio_ch_bufs[ i ], i, 0, vo_audio_buf_size, wav_channels, 1 )
i + 1
}
mjpeg_encoder_write_image( vo_encoder, scr )
mjpeg_encoder_write_audio( vo_encoder, vo_audio_buf, 0, 0 )
mjpeg_encoder_next_frame( vo_encoder )
frame()
}
else
{
frame( 1000 / fps )
}
if !sample_loop && wav_ptr >= wav_size { breakall }
while( get_event() ) { if EVT[ EVT_TYPE ] == EVT_QUIT { halt } }
framecounter + 1;
}
fn gfx_init()
{
vals = new( xsize, 1, FLOAT )
clean( vals )
ftsize = 4096//2048
hftsize = ftsize / 2
ftbufr = new( ftsize, 1, FLOAT )
ftbufi = new( ftsize, 1, FLOAT )
ftmag = new( hftsize, 1, FLOAT )
ftmagsmoo = new( hftsize, 1, FLOAT )
// number of bands in ERB scale
erbsize = 5
erbbuf = new( erbsize, 1, FLOAT)
//erbbufsmoo = new( erbsize, 1, FLOAT)
lightbuf = new( 256, 1, FLOAT)
lightbufsmoo = new( 256, 1, FLOAT)
img2a = clone( img )
img2b = clone( img )
img2c = clone( img )
//copy( img2a, img )
//copy( img2b, img )
//copy( img2c, img )
//resize to blur
//resize(img2c, get_xsize(img)/7,get_ysize(img)/7,-1, RESIZE_COLOR_INTERP2)
//resize(img2c, get_xsize(img),get_ysize(img),-1, RESIZE_COLOR_INTERP2)
set_flags( img2b, RESIZE_COLOR_INTERP2)
framecounter = 0;
}
fn atodb($a){
$db = 20*log10($a)
ret($db)
}
fn dbtoa($db){
$a = pow(10,($db)*0.05)
ret($a)
}
fn audio_callback(
$stream,
$userdata,
$channels,
$frames,
$output_time_in_system_ticks,
$in_channels,
$latency_in_frames )
{
if wav_ptr >= wav_size
{
if !sample_loop
{
ret( 0 )
}
}
$c = 0 while( $c < wav_channels )
{
copy( $channels[ $c ], wav, 0, wav_ptr + $c, $frames, 1, wav_channels )
$c + 1
}
wav_ptr + $frames * wav_channels
if sample_loop
{
if wav_ptr >= wav_size
{
$ff = ( wav_ptr - wav_size ) / wav_channels
$p = $frames - $ff
wav_ptr = 0
$c = 0 while( $c < wav_channels )
{
copy( $channels[ $c ], wav, $p, wav_ptr + $c, $ff, 1, wav_channels )
$c + 1
}
wav_ptr + $ff * wav_channels
}
}
ret( 1 )
}