There's no way to do this automatically. It requires a javascript solution to check the selected fabric options and generate the second color options on the fly. Below is an example, but it's provided, as is, with no guarantees or additional support.
Here's the general idea, starting with the 2 matrix fields. The "Extras" column in the fabric options corresponds to the "Fabric ID" in the Extra Options matrix. So in this instance, Linen would have the colors red, blue, and green. Silk has yellow and blue. etc.

DISCLAIMER: I am not a javascript developer. There will probably be some techniques here that a real JS developer will cringe at. I provide this as a rough example that you'll need to tweak to suit your purposes.
NOTE: You'll need to set protect_javascript to "n" in your EE config settings in order to parse EE tags in your javascript.
Here's the javascript code.
<html>
<head>
{exp:jquery:script_tag}
<script type="text/javascript">
jQuery(document).ready(function($){
{exp:channel:entries entry_id="ADD YOUR ENTRY ID HERE"}
{!-- fabric_options is the short name of the fabric option matrix --}
{fabric_options}
var fabric_options_{option_value}_str = "{extras}";
var fabric_options_{option_value} = fabric_options_{option_value}_str.split(",");
{/fabric_options}
{!-- extra_options is what I've called the color options --}
{extra_options}
{fabric_id} = new Object();
{fabric_id}.option_value = "{option_value}";
{fabric_id}.option_name = "{option_name}";
{/extra_options}
{/exp:channel:entries}
$(".fabric_options").change(function(){
var selected = eval("fabric_options_"+$(this).val());
if (typeof selected == "undefined" || !(selected instanceof Array)) {
<!-- nothing to show -->
}
else
{
// generate a select box.
var s = $("<select name='item_options[extra_options]' />");
for(var i=0; i<selected.length; i++){
$("<option />", {value: eval(selected[i]).option_value, text: eval(selected[i]).option_name}).appendTo(s);
}
// throws the newly minted select box into the HTML
$(".color_select").html(s);
$(".color_select").prepend("Select a Color:");
$(".color_select").append("<br />");
}
});
});
</script>
</head>
<body>
{exp:channel:entries entry_id="ADD YOUR ENTRY ID HERE"}
{exp:cartthrob:add_to_cart_form entry_id="{entry_id}"}
{title}<br />
Choose a Fabric
<select name="item_options[fabric_options]" class="fabric_options"}>
<option value="">Select a Fabric</option>
{fabric_options}
<option value="{option_value}">
{option_name}
</option>
{/fabric_options}
</select><br />
<span class="color_select"></span>
Quantity <input type="text" name="quantity" value="" /><br />
<input type="submit" value="Add to Cart" />
{/exp:cartthrob:add_to_cart_form}
{/exp:channel:entries}
</body>
<html>