I had a minute, so here is the code to get the layers per model space. Now don't look if you don't want to see the code yet.
Edit: Updated code. Wrong syntax on my part.
(defun LayListModelSpace (Doc / LayNameList BlkNameList GetBlockLayers)
(defun GetBlockLayers (BlkDefObj BlkCol)
(vlax-for obj BlkDefObj
(if (not (vl-position (vla-get-Layer obj) LayNameList))
(setq LayNameList (cons (vla-get-Layer obj) LayNameList))
)
(if (= (vla-get-ObjectName obj) "AcDbBlockReference")
(progn
(foreach att (append (vlax-invoke obj 'GetAttributes) (vlax-invoke obj 'GetConstantAttributes))
(if (not (vl-position (vla-get-Layer att) LayNameList))
(setq LayNameList (cons (vla-get-Layer att) LayNameList))
)
)
(if (not (vl-position (vla-get-Name obj) BlkNameList))
(progn
(GetBlockLayers
(vla-Item BlkCol (vla-get-Name obj))
BlkCol
)
(setq BlkNameList (cons (vla-get-Name obj) BlkNameList))
)
)
)
)
)
LayNameList
)
(GetBlockLayers (vla-get-ModelSpace Doc) (vla-get-Blocks Doc))
)
So for you, you would do something like
(setq LayList (LayListModelSpace dbxDoc))
since the routine wants a document object. This will return a list of all the layer names, then you can do what you want with the list.