This is resolved per mikeash - setOpaque. It remains a mystery to me why this is necessary in the case of NSBorderlessWindowMask and not NSTitledWindowMask but I won't loose sleep over it. I'll leave the page here in case anyone follows the link.

When I make a 99% transparent window containing a matrix with a clear background using NSBorderlessWindowMask the window background is visible. If I use NSTitledWindowMask instead the window is clear as glass. I don't know what I need to do to get tranparent borderless windows. This test program demonstrates by generating windows - every other one is created either with NSTitledWindowMask or NSBorderlessWindowMask. It looks like this:

Archived Xcode project (1.4MB)

#import "MyObject.h"

@implementation MyObject

- (id)init
{
    self = [ super init];
	return self;
}

#define MG_COLUMNS		(10)
#define MG_ICONWIDTH	(48)
#define MG_ICONHEIGHT	(48)
#define MG_BORDERWIDTH	( 2)

- (void)awakeFromNib
{
	int i;
	unsigned int styleMask;
	NSString *title;
	NSWindow *window;
	
	for (i = 0; i < 10; i++) {
		title = [[ NSString stringWithFormat:@"window %d", i] retain];
		if (i % 2) {
			window = [[ self createWindowWithStyleMask:NSTitledWindowMask] retain];
			[window setTitle:title];
		} else {
			window = [[ self createWindowWithStyleMask:NSBorderlessWindowMask] retain];
		}
		[ window setAlphaValue:0.9999];
		[ window setBackgroundColor:[[ NSColor redColor] retain]];
		[ window orderFront:self];
	}
}

- (NSWindow *)createWindowWithStyleMask:(unsigned int)styleMask
{
    static NSPoint lastTopLeft = {-10000, -10000};
    NSRect frameRect;
	   NSMatrix *matrix;
	   NSWindow *window;
        
    frameRect.size.width = MG_COLUMNS*MG_ICONWIDTH;
    frameRect.size.height = MG_ICONHEIGHT;
    window = [[ NSWindow alloc] initWithContentRect:NSMakeRect(0, 0, frameRect.size.width+(2*MG_BORDERWIDTH), frameRect.size.height+(2*MG_BORDERWIDTH)) styleMask:styleMask backing:NSBackingStoreBuffered defer:YES];

			    [ window setMovableByWindowBackground:YES];
	    
    frameRect.origin.x = MG_BORDERWIDTH;
    frameRect.origin.y = MG_BORDERWIDTH;
	
    matrix = [[ NSMatrix alloc] retain];
    [ matrix initWithFrame:frameRect mode:NSRadioModeMatrix cellClass:[ NSActionCell class] numberOfRows:1 numberOfColumns:MG_COLUMNS];
    [ matrix setDrawsBackground:YES];
    [ matrix setBackgroundColor:[[ NSColor clearColor] retain]];

    [[ window contentView] addSubview:matrix];

	   if (lastTopLeft.x == -10000) {
		      [ window center];
		      lastTopLeft = [ window frame].origin;
	   }
	   lastTopLeft = [ window cascadeTopLeftFromPoint:lastTopLeft];
    return window;
}
@end