Two questions...(arrays and where to put them) [SOLVED]

#1

hi there,

i have some large arrays like this:


uint8_t spKETTENFETT[715] = {0xe9,0x6f,0xae,0x93,0x3d,0x6a,0xae,0xbf,0xf9,0x4a,0xd3,0x68,0x78,0xfe,0x6e,0xdb,0x5d,0xa2,0xd6,0xfa,0xbb,0x69,0x73,0x89,
0x4a,0xe9,0xaf,0x36,0x24,0x34,0x43,0xa3,0x3f,0x2b,0xe7,0x34,0xb7,0xac,0xfe,0x52,0x5e,0xaa,0xd4,0x1a,0x01,0x16,0xfb,0x44,0xc0,0x37,0xb5,0x08,0xf8,0x36,
0x37,0xfc,0x13,0x44,0x44,0xea,0xc2,0xf2,0x47,0x95,0x19,0x9a,0xaf,0xc7,0x5f,0x79,0x86,0xa9,0xaf,0x2a,0x7f,0xa5,0x99,0x22,0xbe,0x29,0xfd,0x0d,0x59,0x2b,
0xfb,0xa6,0xf4,0x57,0x64,0xad,0x12,0x9b,0xda,0x5f,0x59,0x86,0x88,0xad,0x5a,0x7f,0xe3,0x19,0x26,0xba,0x6a,0xfd,0x03,0x87,0x47,0xc8,0xea,0xf6,0x37,0x90,
0x19,0x29,0xab,0xca,0x9f,0x61,0x67,0x86,0x2e,0x74,0x7f,0x20,0x95,0x1d,0xba,0x90,0x00,0x9f,0xaa,0x31,0xe0,0x9b,0xf4,0xf2,0x0f,0xcb,0x18,0xd5,0xb5,0xda,
0x3f,0xbc,0xa0,0x4e,0x2f,0x6a,0xff,0x70,0x82,0x36,0xbd,0x28,0xfd,0x8d,0x2b,0xd5,0xd4,0x22,0x02,0x1c,0x31,0x86,0x80,0xaf,0xf6,0x10,0xf0,0xd5,0x1c,0x02,
0x7e,0xf4,0x00,0xf4,0x67,0xea,0x1c,0x6e,0x49,0xd2,0xdf,0x15,0xb3,0x6e,0xce,0x6a,0xff,0xb4,0x82,0xb4,0xf5,0xc8,0x01,0xd3,0x2b,0x1a,0x60,0x7a,0x41,0xf3,
0x0f,0x43,0x68,0x57,0x23,0xd1,0x9f,0x28,0xfb,0xb5,0xc6,0x26,0x7f,0x2d,0x4f,0x59,0x6c,0x45,0xfd,0xdd,0x3c,0xeb,0x48,0x3c,0x02,0x7c,0x95,0x83,0x80,0x6f,
0x72,0x09,0xb0,0xd9,0xb6,0xfa,0xa3,0x68,0x31,0xb5,0xc6,0xe2,0x67,0x25,0xd5,0x33,0x62,0x99,0x5f,0xa4,0x77,0xf1,0x8c,0x55,0xfe,0x28,0xaf,0x48,0x6b,0x4e,
0xfb,0xb3,0xf8,0x42,0xa9,0xd9,0xed,0x2f,0xfc,0x8b,0x24,0x1a,0x97,0xbf,0xf0,0x0b,0x12,0x5f,0x44,0xfe,0x80,0x3a,0xcc,0xa3,0xb1,0xf9,0x2b,0x1e,0x35,0xb7,
0x5a,0xe3,0x2f,0x6a,0x4c,0xcd,0x57,0xad,0xbf,0xaa,0x4d,0x96,0x6c,0xbc,0xfe,0x26,0xb7,0x84,0xb3,0xd1,0xfa,0x87,0xa8,0x52,0xb5,0x46,0xeb,0x9f,0xd2,0xdd,
0x43,0x1b,0x8d,0x7f,0x08,0x15,0x6f,0x6b,0x94,0xfe,0x44,0x23,0xba,0x38,0xb1,0xfa,0x59,0x69,0xb5,0x8a,0x44,0xea,0xa7,0x6d,0xd4,0x32,0x6c,0x85,0xdf,0x9a,
0x4f,0xd6,0xac,0xdd,0xfe,0xcc,0xaf,0x48,0xba,0x6e,0xfb,0x2b,0xbb,0x61,0xac,0x2a,0xed,0xaf,0x74,0x9a,0x29,0xeb,0x84,0xdf,0xab,0x74,0x0f,0x4d,0xab,0x7e,
0xee,0xbc,0x2b,0xcc,0xae,0xfa,0x33,0xb0,0x9c,0xc8,0x38,0x02,0xf8,0xda,0x7d,0xfc,0x5d,0x86,0xa6,0xd8,0xa2,0xf5,0x77,0x1b,0x69,0xa6,0x0d,0xd7,0xdf,0x6d,
0xa6,0x99,0xd5,0x5c,0x7f,0xf7,0x19,0xa6,0x56,0x33,0xfc,0xd5,0xbb,0x98,0x46,0x65,0xf4,0x17,0x53,0xe4,0x6a,0x51,0xd4,0x5f,0xc2,0x4b,0xa4,0xda,0x23,0xc0,
0x61,0x93,0x08,0xf8,0xba,0x16,0x01,0xdf,0xe5,0xa4,0x7f,0xc0,0xf4,0x70,0x5d,0xd9,0xfe,0x24,0x33,0xdd,0xf2,0x55,0xfb,0x2b,0xcf,0x14,0xf5,0x55,0xe9,0xaf,
0xd8,0x5b,0x25,0x36,0xa5,0xbf,0x22,0x6b,0x95,0xd8,0x54,0xfe,0x8a,0xa3,0x44,0x63,0xd3,0xf8,0xab,0xc8,0x50,0xf5,0x57,0xeb,0xaf,0x2c,0x23,0xdc,0x57,0x8f,
0xbf,0xd2,0x8c,0x48,0x5b,0xdd,0xfe,0x8a,0x23,0xa2,0x74,0x55,0xf9,0x33,0x0d,0xcf,0xd4,0x55,0xe6,0x6f,0x20,0x2c,0xcb,0x93,0x10,0xe0,0xdb,0x2a,0x01,0x2c,
0xe9,0xd0,0xfe,0xe1,0x99,0x74,0x7b,0x56,0xfb,0x67,0x54,0xa4,0xad,0xc7,0xed,0x1f,0x51,0x91,0xb6,0x1e,0xb7,0x7f,0x44,0x45,0xde,0x5a,0x34,0xfe,0xe9,0x94,
0x78,0x73,0xd1,0xf8,0x87,0x12,0xb1,0x8d,0x45,0xe3,0x6f,0x5c,0xac,0xc6,0x57,0x95,0xbf,0x0a,0xb5,0xf4,0x58,0x19,0xfe,0x8a,0x4a,0xdc,0x73,0x26,0x03,0x36,
0x74,0x63,0x7f,0x03,0xe1,0x5e,0xbe,0x08,0xfd,0x5c,0x4c,0x5a,0x47,0x66,0xf7,0xe7,0xb4,0x41,0xa5,0xb2,0xc3,0x9f,0x72,0xa8,0x84,0xd4,0x09,0x7f,0xc8,0x92,
0x1a,0xd6,0x34,0xfd,0xb1,0x4a,0x48,0xd9,0xd2,0xf6,0xe7,0xe0,0xa1,0xa5,0x4b,0xdb,0xdf,0xac,0xba,0xb5,0xac,0x69,0x7f,0xf7,0x62,0xda,0xfa,0xba,0xfc,0xdd,
0x8b,0xe9,0xd8,0x9b,0xf4,0x0f,0x2f,0x22,0xe3,0x8b,0xc3,0x3f,0x82,0x31,0x4e,0x2c,0x36,0xff,0x88,0x8a,0xb4,0xb9,0x58,0xfc,0xdd,0x31,0xda,0xe7,0x22,0xf4,
0x07,0xea,0xbe,0x67,0x8e,0x11,0xf0,0xad,0x17,0x01,0xbe,0x99,0x41,0xc0,0xd5,0x3b,0x08,0x38,0xfa,0x86,0xfc,0xc3,0xbf,0xb3,0xa4,0x5b,0x00,0x00,0x00,0x00,
0x78};

uint8_t spZUCKER[122] = {0x08,0x58,0xaa,0x0d,0xc8,0x3f,0x5c,0x7b,0x66,0xd8,0x21,0xff,0x0c,0x1d,0x91,0x61,0xc7,0x00,0xd3,0x77,0xb8,0x7f,0xfa,0x4e,0x8f,0x48,
0x52,0xfe,0xac,0x5c,0x35,0xc3,0xf1,0xf8,0x6d,0x2e,0xf1,0x2a,0x39,0xed,0xb7,0x2d,0xc4,0x3b,0x15,0x9b,0x5f,0x77,0x93,0x68,0x57,0x64,0xfe,0xd6,0xdd,0x33,
0x35,0x71,0xf9,0xfb,0x8d,0x08,0xf3,0x24,0xed,0x6f,0x89,0x54,0x2a,0x62,0xad,0x3f,0x3a,0x0b,0x6f,0x96,0x73,0xfe,0x14,0x2c,0x62,0xd8,0xf1,0xf9,0x53,0x34,
0xcf,0x16,0xc5,0xeb,0x4f,0xd1,0x2c,0x5b,0x14,0x8f,0x3f,0x7a,0xd7,0x68,0xb2,0x1d,0xfe,0xe4,0x5c,0xbc,0xd4,0x89,0xf8,0x8b,0x0f,0x91,0x56,0xc7,0xe8,0xef,
0xa9,0x5d,0xd6,0x33,0xc3,0x03};

uint8_t *words[2] = {spKETTENFETT, spZUCKER};

and i adress them via the words pointer. this all works beautifully.

currently this all resides in my local data of the object.

my two questions are:

1) how do i force those arrays into sdram? i tried using attribute ((section (".sdram"))) but i always get an fpermissive error... if somebody could guide me here that would be great. i still need to be able to point to those arrays with the words pointer.

2) is there a way to put those arrays into a separate object and access the data from the main object? i would put all the arrays and the words pointer into the separate object.

Custom File Loader and Structs
#2

i think i had the first issue when doing elements.... but honestly it was a while ago...
as i remember, whilst you can allocate the memory in sdram you cannot assign constant data to it, though i can't remember if i got compilation errors when trying, or if the issue was the constant data was also ending up in code memory too....
(sorry, dont have time at the mo, to go verify exactly what the issue was)

anyway the upshot, and perhaps a solution for your, was i did this as two stages, basically dynamic sdram memory allocation (available in 1.0.12) and then reading the resource data from a file , in the end it was quite a 'tidy' way to do it... you'll find the code in the init section of lmnts... and the loadElementsData in the firmware (make sure you look at 1.0.12 code base as its different in the newer dev version)

as i said though, this is 'from memory' so i may be mis-remembering some parts of this

(2) sure you can do this, but you end up either a) hard coding the object name, or b) need the user to type in the object name (as done with tables)
but if this is just implementation, that the user does not need to see, then why bother with axoloti objects? just create a C++ class, and then use the singleton pattern (or variation of), really you only need axoloti objects when its something you need the user to interact with.

this is pretty much what goes on with all the MI objects, you have axo objects as the UI, but the really MI code is completely separate.

the only messy thing is , in 1.0.12 there is no ability to add extra compile modules, so you end up including as 'headers' (as I did in my push object), in the next release this changes, as ive created a concept of compilable (user) modules.

#3

thanks for your answer!

it is custom words and phrases that a user has to enter (generated by BlueWizard: https://github.com/patrick99e99/BlueWizard) so i think i have to stick with an axoloti object. or i just create a .h file that the user can edit to his liking. will try to go that route. but that does not allow me to put it in sdram...

loading from a file seems not possible, because i don't know the length of the arrays beforehand, they are also defined by the user and thus would have to be saved in the file as well. hmm, wait you said there is dynamic memory allocation for sdram now, phhuhh, WAYYYY over my head for the moment...thanks for your explanations though.

#4

@lokki, I hav been trying to simply use the "init" "Edit" thing on a table and paste an array into that, trying to load it into sd-ram. And I kept getting the same error. Sorry dont remember the exact error, but it wouldnt let med access the sd-ram like that.

BUT, there is a way you can store it in a file on SD-card that you can load into SD-Ram.

I have hacked sirsicksik's object called sss/harmony/scaleGenerator to suit my own needs. You can probably do the same. It basically saves an array from the object to sd card. And then use another object to load it again.

The SSS object has got the array in local data, and as SOON as you go live it stores the array to the sd-card at a given location. And then you load the array into a SINGLE object.... The object that LOADS the array is called sss/harmony/ScaleBank. UPDATE: Sorry its not the sss/harmony/ScaleBank that load the aray form sd card. But its one of the object in sss's harmony folder.

#5

thanks @jaffasplaffa that is a beginning. i will see if i can add more then one array in a file this way. you see the length is also user specific, so i am not sure if this will work out.

#6

Yeah check them out, it might work :slight_smile:

Ah yes I the length thing....... Have been thinking hard about how to do that, setting it dynamically for a sampler I have been working on. Used drjuustice filesize object as an easy solution to get the filesize and then set the compensation according to the tablesize. But not the best since youll use more sdram than you need.

I have not tried this yet, but I think its possible to do. You can basically set the tablesize to anysize(also not power of 2) by some manipulation of some of the table variables, See the sir sick sik object.

And combining this technique with drjustice "filesize" object, I think it might be possible to set the table sixe according to the file you want to load in to the table. My thought was Order Of Execution:
1. Get files size
2. Set tablesize to the same as file size.
3. Load file into table.

This works in Pure Data. So it must be possible on Axoloti too. Have thought about it many times, but just never tried to implement it yet.

#7

i think you can also do this in one object, there you can control order of execution by putting it in code :slight_smile: yeah i will have to think about the best options. i want to load multiple arrays (say 100) of different sizes (between 100 and 500) and i must access them via a pointer so that i can dynamically change which array is read.

phuuh! maybe it would be easier to put all the data into a MEGA array and save a second array with all the onsets.

#8

@lokki

I just tried editing the regullar table. Instead of using the combo attribute to set the size by power of 2, I changed it to a spinner attribute.... Which lets you set ANY size you want :slight_smile: And it compiles so I think that actually works, all though I didnt test it:
Spinner tablesize 1.0 .axp (1.7 KB)

BUT When wanting to set the tablesize dynamically, which you do in local code, I see there might be some conflicts, we cant acces the local data when patch is live. If you can get the filesize in the local code and process that before th etablesize is set, then it should be possible. But havent tried this.

#9

@lokki ,
if you look at the elements code you will see it dynamically allocates the sdram size... using sdram_malloc()

you can also get a files size by using f_stat() , takes a filename, a fileinfo structure (pointer), the later contains the filesize.

just be aware that you don't want to go around alloc/freeing memory frequently, as its a simplistic memory manager - but its fine for loading data at startup of patch.

#10

thanks, i will have to dive into this, will take some time.

#11

Definitely also looking into the suggestions from technobear.

I would really like to get " set tablesize according to filesize working".

#12

ok i created a custom object that writes two arrays, one with all the data, one with all the offsets (sizes of different data) to sdcard. i did this all with attributes, so i can insert all the data into a textfield, select the size of the data etc. this works! though i have a strong feeling, once i try to put in more data into the array then what fits into sram i will have to figure out another way :slight_smile:

thanks @jaffasplaffa and @thetechnobear for the suggestions, now on to allocating dynamically based on file size :slight_smile:

#13

Great to hear :slight_smile: 20 characters

#14

hey @thetechnobear i have it working with dynamic sdram allocation!!! thanks heaps, this is great.
i only wrote a small object to try it, now on to implementing it on the real thing. cool stuff.

#15

@lokki Sounds great. Is it also possible to set the tablesize while the patch is live? Like ofr example if one used it for a sampler nad wanted to change sample when being live?

#16

probably not a good idea if you look at what @thetechnobear wrote a couple of posts above.
sdram_malloc() should only be used in the init section of an object...

#17

ok, @thetechnobear, i have one other question, if you don't mind. as i wrote i was successful in mallocing some sdram space. my object works great with this. since i though about loading multiple instances that would use the same data, can i also do the
[parent -> polyIndex][0] trick with the array i allocated dynamically to access the data from more than one object? or do i have to initialise a pointer via
[attr_poly][LENGTH] attribute ((section (".sdram"))) and point that to the location of the array that i malloced in sdram? either way, i cannot get it to work :frowning: but maybe this is also not possible. i could not see from your code how/if you do that in the elements code. what happens if you open more then one instance of those?

#18

ok, the thing to realise is these 'tricks' are only really necessary because each instance of an object is created as a separate class, and so access is restricted, because that class name is the instance name of the object... so you cant reference it easily.

so for complex objects, i think its much easier to just define a new class in a header file, which you then include (object editor supports include)
that way you know what the class name is called, so you can easily implement a singleton pattern. (which is basically just a static pointer wrapped in a class)

it perhaps sounds complex, but in reality its a couple of lines of code... and also provides a perfect place to put 'common' code e.g. loading your data from the file.

(its a pity i didn't structure the Push object like this, not quite sure why i did it using C functions... was a long time ago :wink: )

tip... what i do when looking at this kind of thing, is look at the xpatch.cpp (run it thru a pretty print), it then becomes fairly clear the C++ 'syntax issues' your seeing... I guess that's why I also don't think exclusively in terms of axo object, and more just in how it looks in C++.

lmnts, nah, it (stupidly) loads the data per instance... because i kind of knew you could only have one instance of it... partly due to cpu, and iirc, the MI code has some statics which cause issues with multiple instances.

#19

thanks again. hmm again over my head for the moment. have to read up on that singleton pattern. yeah it sounds complex because i am not a programmer really :slight_smile: so i would put all my loading data code and other common stuff into the header file. and create a class where i put in all the static pointers i need and it would only be called once when i create multiple instances? sounds great. phuuh.

i looked at the patch.cpp and i can see all the classes there. so basically something like this should work for my header file?

;class lpc_common{
public:
static uint8_t *lpc_data;
static uint16_t *onset_data;
static uint32_t *word_idx;
static uint8_t wordcount = 0;

void dataInit () {
FIL lpcfile, onsetfile;

uint32_t lpcfile_len, onsetfile_len;

FRESULT err;

UINT bytes_read;

err = f_open(&lpcfile, "0:/lpcdata.sns", FA_READ | FA_OPEN_EXISTING);

if(err != FR_OK) {
LogTextMessage("Unable to open lpcdata, is the file on the sdcard?");
return;
}

lpcfile_len = f_size(&lpcfile);

lpc_data = (uint8_t *)sdram_malloc(lpcfile_len);

// Load lpcdata into sdram
int remaining_bytes = lpcfile_len;
int offset = 0;

while (remaining_bytes > 0) {
if (remaining_bytes > sizeof(fbuff)) {
err = f_read(&lpcfile, fbuff, sizeof(fbuff), &bytes_read);
if (bytes_read == 0)
break;
memcpy((char*)(&lpc_data[0]) + offset,(char *)fbuff,bytes_read);
remaining_bytes -= bytes_read;
offset += bytes_read;
} else {
err = f_read(&lpcfile, fbuff, remaining_bytes, &bytes_read);
memcpy((char*)(&lpc_data[0]) + offset,(char *)fbuff,bytes_read);
remaining_bytes = 0;
}
}

if (err != FR_OK) {
LogTextMessage("Failed reading lpcdata file, aborting...\n");
return;
}

err = f_close(&lpcfile);
if (err != FR_OK) {
LogTextMessage("Failed closing lpcdata file, aborting...\n");
return;
}

//load onsets into sdram

err = f_open(&onsetfile, "0:/onsets.sns", FA_READ | FA_OPEN_EXISTING);

if(err != FR_OK) {
LogTextMessage("Unable to open onsets, is the file on the sdcard?");
return;
}

onsetfile_len = f_size(&onsetfile);

//16bit only needs half the size because f_size reports 8bits(bytes) not 16bit
onset_data = (uint16_t *)sdram_malloc(onsetfile_len/2);

//16bit uses two slots, equals length reported by f_size, since we are writing one byte at a time
remaining_bytes = onsetfile_len;
offset = 0;

while (remaining_bytes > 0) {
if (remaining_bytes > sizeof(fbuff)) {
err = f_read(&onsetfile, fbuff, sizeof(fbuff), &bytes_read);
if (bytes_read == 0)
break;
memcpy((char*)(&onset_data[0]) + offset,(char *)fbuff,bytes_read);
remaining_bytes -= bytes_read;
offset += bytes_read;
} else {
err = f_read(&onsetfile, fbuff, remaining_bytes, &bytes_read);
memcpy((char*)(&onset_data[0]) + offset,(char *)fbuff,bytes_read);
remaining_bytes = 0;
}
}

if (err != FR_OK) {
LogTextMessage("Failed reading onset file, aborting...\n");
return;
}

err = f_close(&onsetfile);
if (err != FR_OK) {
LogTextMessage("Failed closing onset file, aborting...\n");
return;
}
//calculate word indexes from wordlengths and put into sdram
word_idx = (uint32_t*)sdram_malloc(onsetfile_len/2);
int i;
for(i=0;i<sizeof(word_idx);i++){
if (i == 0) word_idx[0] = 0;
else {
word_idx[i] = onset_data[i-1] + word_idx[i - 1];
}
}

wordcount = sizeof(word_idx);

}

}

and i can then access the pointers via lpc_common.lpc_data and the file loading via lpc_comon.dataInit() ?
probably not, but maybe something in that direction?

thanks for your time and support, really appreciated.

#20

funny how in the preview the code tags work, but in the posted example they don't really...